diff --git a/Makefile b/Makefile index 54f129032..0c5ed6d54 100644 --- a/Makefile +++ b/Makefile @@ -39,14 +39,14 @@ sw/toolchain/riscv-gnu-toolchain: sw/toolchain/riscv-gnu-toolchain.version sw/toolchain/llvm-project: sw/toolchain/llvm-project.version mkdir -p sw/toolchain - cd sw/toolchain && git clone https://github.com/boboloiono/llvm-project.git + test -d $@/.git || git clone https://github.com/boboloiono/llvm-project.git $@ cd sw/toolchain/llvm-project && \ git checkout `cat ../llvm-project.version` && \ git submodule update --init --recursive --jobs=8 . sw/toolchain/riscv-opcodes: sw/toolchain/riscv-opcodes.version mkdir -p sw/toolchain - cd sw/toolchain && git clone https://github.com/boboloiono/riscv-opcodes.git + test -d $@/.git || git clone https://github.com/boboloiono/riscv-opcodes.git $@ cd sw/toolchain/riscv-opcodes && \ git checkout `cat ../riscv-opcodes.version` && \ git submodule update --init --recursive --jobs=8 . diff --git a/hw/ip/snitch/src/riscv_instr.sv b/hw/ip/snitch/src/riscv_instr.sv index a0f06739a..0a7719209 100644 --- a/hw/ip/snitch/src/riscv_instr.sv +++ b/hw/ip/snitch/src/riscv_instr.sv @@ -1078,15 +1078,15 @@ package riscv_instr; localparam logic [31:0] P_FLH_RRPOST = 32'b1010000??????????001?????1111011; localparam logic [31:0] P_FLW_RRPOST = 32'b1010000??????????010?????1111011; localparam logic [31:0] P_FLD_RRPOST = 32'b1010000??????????011?????1111011; - localparam logic [31:0] MSETMTYPE = 32'b1000001??????????111000001010111; - localparam logic [31:0] MSETTN = 32'b100001000000?????111?????1010111; - localparam logic [31:0] MSETTM = 32'b100001000001?????111?????1010111; - localparam logic [31:0] MSETTK = 32'b100001000010?????111?????1010111; - localparam logic [31:0] MSETMTYPEI = 32'b1000010???11?????111000001010111; - localparam logic [31:0] VTFMM_TVV = 32'b1111001??????????001????01110111; - localparam logic [31:0] VTFMM_ALT_TVV = 32'b1111001??????????001????11110111; - localparam logic [31:0] VTMMU_TVV = 32'b1111001??????????000????01110111; - localparam logic [31:0] VTMMS_TVV = 32'b1111001??????????000????11110111; + localparam logic [31:0] MSETTN = 32'b100010000000?????111?????1010111; + localparam logic [31:0] MSETTM = 32'b100010000001?????111?????1010111; + localparam logic [31:0] MSETTK = 32'b100010000010?????111?????1010111; + localparam logic [31:0] MSETMTYPEI = 32'b100?100??100000??111000001010111; + localparam logic [31:0] MSETMTYPE = 32'b1000010??????????111000001010111; + localparam logic [31:0] VTFMM_TVV = 32'b1111001??????????001???001110111; + localparam logic [31:0] VTFMM_ALT_TVV = 32'b1111001??????????001???011110111; + localparam logic [31:0] VTMMU_TVV = 32'b1111001??????????000???001110111; + localparam logic [31:0] VTMMS_TVV = 32'b1111001??????????000???011110111; localparam logic [31:0] VTLE8 = 32'b0001001??????????111000000000111; localparam logic [31:0] VTLE16 = 32'b0011001??????????111000000000111; localparam logic [31:0] VTLE32 = 32'b0101001??????????111000000000111; diff --git a/hw/ip/spatz/src/generated/spatz_pkg.sv b/hw/ip/spatz/src/generated/spatz_pkg.sv index 41859ae21..f6676a2cc 100644 --- a/hw/ip/spatz/src/generated/spatz_pkg.sv +++ b/hw/ip/spatz/src/generated/spatz_pkg.sv @@ -95,16 +95,19 @@ package spatz_pkg; localparam int unsigned NrPhysicalTile = 16; // Tile Dimension localparam int unsigned OPEComputeEdge = 8; - // The ratio of Tile Dimension and OPE FMA array size + // Number of compute-edge groups along one tile edge localparam int unsigned NrAccPerComputeElem = 2; localparam int unsigned TileEdge = OPEComputeEdge * NrAccPerComputeElem; // The controller further limits tk to 32/SEW, or to one for 64-bit accumulation. localparam int unsigned KMAX = (AccElemWidth <= 32) ? (32 / 8) : 1; - // Datapath between VLSU and OPE - localparam int unsigned TileDataWidth = TileEdge*AccElemWidth; + // Maximum storage required for one complete Tile row, in bytes. + localparam int unsigned TileRowBWidth = TileEdge * AccElemBytes; + // Fixed word width of the datapath between VLSU and OPE, in bytes. + localparam int unsigned TileWordBWidth = TileEdge * AccElemBytes; typedef logic [$clog2(TileEdge)-1:0] tile_dim_t; typedef logic [$clog2(TileEdge+1)-1:0] tile_count_t; + typedef logic [$clog2(TileWordBWidth)-1:0] tile_word_elem_idx_t; typedef logic [$clog2(NrPhysicalTile)-1:0] mt_t; typedef struct packed { logic tile_valid; @@ -146,19 +149,22 @@ package spatz_pkg; typedef logic [N_FU*ELENB-1:0] vrf_be_t; typedef logic [N_FU*ELEN-1:0] vrf_data_t; - typedef logic [TileDataWidth-1:0] tile_row_t; + typedef logic [8*TileRowBWidth-1:0] tile_row_t; + typedef logic [8*TileWordBWidth-1:0] tile_word_t; typedef struct packed { mt_t idx; tile_dim_t row; + tile_dim_t col; tile_count_t elems; vew_e tew; } tile_r_req_t; typedef struct packed { mt_t idx; tile_dim_t row; + tile_dim_t col; tile_count_t elems; vew_e tew; - tile_row_t data; + tile_word_t data; } tile_w_req_t; // Instruction ID diff --git a/hw/ip/spatz/src/ope/docs/1. overview.md b/hw/ip/spatz/src/ope/docs/1. overview.md new file mode 100644 index 000000000..97b9ef0c5 --- /dev/null +++ b/hw/ip/spatz/src/ope/docs/1. overview.md @@ -0,0 +1,35 @@ +# Spatz VME: Study Overview + +## Research Focus + +This study integrates the Zvt version 0.4 Vector Matrix Extension into the Snitch–Spatz system. Spatz already provides a compact vector register file (VRF), vector arithmetic, and access to a banked tightly coupled data memory (TCDM). The extension adds matrix configuration, an Outer Product Engine (OPE), and matrix tile state. Matrix instructions take their source operands from the VRF and accumulate into tiles, while existing vector instructions remain available for data preparation and result processing. + +The central question is how to make matrix arithmetic useful in a complete GEMM kernel on a compact vector processor. Input loads, register allocation, instruction supply, tile transfers, and output processing can limit performance even when the arithmetic array is busy. The study considers FP32-to-FP32 and widening FP16-to-FP32 computation and examines how tile layout and instruction scheduling affect these costs. + +## Design and Evaluation Questions + +- **Operand and result movement:** How much data moves from TCDM to the VRF, from the VRF to the OPE, and from the tiles back to memory or vector registers? When does direct tile storage avoid a VRF transfer, and when is a tile-to-vector move useful for post-processing? +- **Shared resources:** How do the VAU and OPE share VRF access, and how do the controller and execution units preserve dependencies between vector and matrix instructions? +- **Tile geometry:** How do the architectural tile edge (TE), physical compute edge (CE), VRF word width, and Tile interface width affect instruction count, execution time, and transfer granularity? +- **Kernel mapping:** How do output tile layouts, vector register groups, FP16 operand packing, and store schedules affect input reuse, register lifetime, and overlap between computation and output transfer? +- **Measured behavior:** Which predicted costs appear in cycle-accurate simulations, and which stalls arise from instruction supply, memory accesses, VRF contention, or accumulator dependencies? + +## Scope of the Work + +The hardware contribution is the integration of the OPE and tile data paths into Spatz, including matrix configuration, instruction dispatch, dependency management, and coordination with the double-bandwidth VLSU. The software contribution is a set of matrix kernels and scheduling choices that exercise the FP32 and widening FP16 modes. An analytical model separates instruction counts, element transfers, and byte traffic before the measured performance is discussed. + +TE describes the tile visible to a matrix instruction; CE describes the physical compute array that processes it. Changing CE can change the number and timing of physical execution beats without changing the number of architectural matrix instructions for a fixed TE and workload. Likewise, a tile-to-vector move uses the VRF path, while a tile load or store uses the VLSU Tile interface. The analysis treats these as distinct transfers. + +## Chapter Roadmap + +1. **Introduction** states the workload problem, research goals, and hardware and software contributions. +2. **[Background and Related Work](<2. Background and Related Work.md>)** introduces matrix dataflows, vector data movement, SSR, MX, O-POPE, and matrix engine integration choices. It compares where operands and partial sums reside and how results leave the compute unit. +3. **[System Overview and Spatz VME Architecture](<3. System Overview and Spatz VME Architecture.md>)** introduces Snitch, Spatz, the VRF, VAU, OPE, VLSU, and TCDM. It follows instruction dispatch and completion, then defines the software-visible Zvt configuration, tiles, arithmetic, and transfer operations. +4. **[Spatz VME Microarchitecture](<4. Spatz VME Microarchitecture.md>)** describes the implemented controller, scoreboards, shared VRF arbitration, distributed OPE, accumulator behavior, and VLSU Tile path. +5. **[System-Level and ISA Exploration](<5. System-Level and ISA Exploration.md>)** examines VLEN, TE, CE, precision, instruction demand, accumulator capacity, and the distinct VRF and Tile transfer paths. It separates analytical design points from the implemented configuration. +6. **[Kernel Design](<6. Kernel Design.md>)** will describe FP32 and packed FP16 operand layouts, output tile mappings, register allocation, configuration changes, and store schedules. This chapter is reserved for later writing. +7. **Experimental Methodology** specifies the hardware configuration, instruction cache and buffer, workloads, baselines, correctness checks, simulation procedure, and definitions of cycles and utilization. +8. **Results and Discussion** evaluates the kernel and hardware choices against the analytical predictions, using traces and measured cycles to explain bottlenecks and limitations. +9. **Conclusion and Future Work** summarizes the supported findings and identifies remaining architectural and software opportunities. + +The analytical model establishes expected work and traffic; the experimental results determine whether a particular mapping or schedule improves execution on this implementation. Statements about issue limits and transfer cycles therefore refer to the specified register mapping and data path rather than to a general Zvt ISA limit. diff --git a/hw/ip/spatz/src/ope/docs/2. Background and Related Work.md b/hw/ip/spatz/src/ope/docs/2. Background and Related Work.md new file mode 100644 index 000000000..1b624699c --- /dev/null +++ b/hw/ip/spatz/src/ope/docs/2. Background and Related Work.md @@ -0,0 +1,31 @@ +# 2. Background and Related Work + +## 2.1 Matrix Workloads and Data Reuse + +Matrix multiplication appears both as a standalone kernel and as one stage of a longer computation. Inputs may require packing, transposition, or conversion before multiplication. Outputs may be consumed by bias addition, scaling, activation, or quantization. The cost of these stages depends on where the operands and partial sums reside and how results reach the next operation. Arithmetic throughput alone therefore does not describe the cost of a complete matrix workload. + +For a matrix product `C[M,N] += A[M,K] × B[K,N]`, an inner-product mapping computes each output element by reducing one row of `A` against one column of `B`. A row-wise vector mapping broadcasts an element of `A` across a row of `B` and updates a corresponding row of `C`; it offers contiguous vector accesses for row-major `B` and `C`, but keeping several output rows live consumes vector registers. An outer-product mapping uses a column segment of `A` and a row segment of `B` at each reduction step to update a two-dimensional region of `C`. Each input element is reused across one dimension of that region, while the output partial sums must remain available throughout the reduction. + +Mixed-precision matrix multiplication reduces input storage and transfer width while retaining a wider accumulator. A widening FP16-to-FP32 operation, for example, consumes FP16 multiplicands and accumulates into FP32 results. The wider accumulator limits rounding during the `K` reduction, while input conversion and packing still contribute to the overall workload. These properties motivate examining memory traffic, register pressure, and output handling alongside the compute datapath. + +## 2.2 Data Movement in Compact Vector Processors + +A vector instruction applies one operation to multiple elements and amortizes instruction fetch and decode over the vector length. A vector register file (VRF) can also retain inputs and intermediate values close to the functional units, reducing accesses to shared L1 memory. In its floating-point Spatz cluster, Perotti et al. analyze this trade-off and use a compact, latch-based 2 KiB VRF per vector processing element. A small VRF lowers storage cost, but the available register capacity and ports constrain how many matrix operands and output partial sums can remain live at once. [Spatz paper](https://arxiv.org/html/2309.10137#S4) + +Regular matrix loops also spend scalar instructions on address updates and explicit data transfers. Stream Semantic Registers (SSR) associate registers with configured memory streams, removing many explicit loads and stores from the inner loop of a scalar core. SSR primarily changes how operands are supplied; the storage of matrix partial sums depends on the compute architecture coupled to those streams. [SSR paper](https://arxiv.org/abs/1911.08356) + +These approaches trade instruction traffic against data movement and local storage. A longer vector or a more aggressive loop unroll can reduce control overhead, but may occupy more registers or enlarge the instruction stream. A matrix instruction can describe more arithmetic work, yet still require packing, register transfers, and result writeback. The useful comparison is therefore the movement of `A`, `B`, and `C` through memory, the VRF, and the compute unit over the full kernel. + +## 2.3 Matrix Engine Integration + +Matrix extensions differ in how much of the vector datapath they reuse. MX extends Spatz by retaining the existing VRF and functional units, adding operand broadcast and a compact buffer near the FPUs to increase reuse during matrix multiplication. This limits new hardware but keeps matrix arithmetic tied to the throughput and scheduling of the vector functional unit. [MX paper](https://arxiv.org/abs/2401.04012) + +Architectures with separate tile state keep output partial sums outside the ordinary vector registers. Arm SME, for example, provides ZA matrix storage with tile-slice transfers to vector registers and memory. Intel AMX provides tile registers and a tile-multiplication engine. These designs expose different tile organizations and transfer operations, so their peak arithmetic rates alone are not a sufficient basis for comparing full kernels. [Arm SME overview](https://learn.arm.com/insights/sme/) · [Intel AMX overview](https://www.intel.com/content/www/us/en/products/docs/accelerator-engines/what-is-intel-amx.html) + +An outer-product engine can exploit the reuse of each operand across a two-dimensional compute array. O-POPE studies a pipelined implementation that uses FPU pipeline registers to hold intermediate results and reduce dedicated buffering. That work concerns the compute engine itself; integrating such an engine into a vector processor additionally requires instruction dispatch, register dependencies, memory access, and paths for moving tile results to later vector operations. [O-POPE paper](https://arxiv.org/abs/2606.02333) + +## 2.4 Position of This Work + +This work integrates a separate Outer Product Engine (OPE) into Spatz while retaining the VRF as the source of matrix operands. The OPE holds two-dimensional tile results, and the existing vector datapath remains available for input preparation and output processing. Tile transfers to and from memory provide another path for results that do not require a vector post-processing step. The design consequently combines a dedicated matrix compute and accumulator path with shared vector storage and a common instruction-control domain. + +The questions addressed by this integration are concrete: how operands reach the OPE without adding dedicated VRF operand ports; how vector and matrix instructions remain ordered while using different result storage; and when tile results should pass through the VRF instead of moving directly to memory. Chapter 3 introduces the Snitch–Spatz system and the software-visible Zvt model. Chapter 4 describes the corresponding hardware paths and dependency mechanisms. diff --git a/hw/ip/spatz/src/ope/docs/3. System Overview and Spatz VME Architecture.md b/hw/ip/spatz/src/ope/docs/3. System Overview and Spatz VME Architecture.md new file mode 100644 index 000000000..a1f515464 --- /dev/null +++ b/hw/ip/spatz/src/ope/docs/3. System Overview and Spatz VME Architecture.md @@ -0,0 +1,67 @@ +# 3. System Overview and Spatz VME Architecture + +This chapter introduces the Snitch–Spatz system, follows an instruction from the scalar core through Spatz, and describes the software-visible matrix state used by this work. The original Spatz instruction path and functional units are drawn from the [Spatz architecture paper](https://arxiv.org/html/2309.10137#S4). The matrix instruction semantics follow [Zvt version 0.4](https://github.com/riscv/riscv-isa-manual/blob/zvt/src/unpriv/zvt.adoc). Chapter 4 describes how the implementation realizes those semantics. + +## 3.1 Snitch–Spatz Core Complex and Shared Memory + +A Snitch scalar core and a Spatz vector processor form a core complex. Snitch executes scalar control instructions, including branches, loop-index updates, and address calculations. Spatz executes data-parallel instructions. They communicate through the CORE-V X-Interface (XIF): Snitch identifies an instruction for offload and supplies the instruction word and any scalar operands, while Spatz decodes and executes the vector operation. This division allows a small scalar core to control a wider datapath without implementing the vector functional units inside the scalar pipeline. [Spatz paper, Section IV-A](https://arxiv.org/html/2309.10137#S4.SS1) + +Core complexes access a banked, tightly coupled data memory (TCDM) through the cluster interconnect. Conventional vector loads move data from TCDM into the VRF, and vector stores move it back. The cluster can also bring data from external memory into TCDM through its DMA path. The Zvt tile load and store instructions introduced in this work add a route between TCDM and matrix tile state, so their payload need not pass through the VRF. + +The [default configuration for this study](../../../../../system/spatz_cluster/cfg/spatz_cluster.doublebw.dram.hjson) has a 128 KiB TCDM divided into 16 banks, VLEN = 512 bits, four vector FPUs, and eight 64-bit Spatz memory ports. The Spatz paper evaluates a two-core-complex cluster with the same total TCDM capacity and bank count, but its illustrated Spatz instance uses a number of memory ports that matches the number of functional units. The port count and bandwidth statements below refer to the present double-bandwidth configuration, not to the paper's baseline cluster. + +## 3.2 Instruction Flow Across Snitch and Spatz + +Snitch fetches an RVV or Zvt instruction in program order and performs the predecode needed to recognize an offloaded operation. It forwards the instruction and any required scalar register values over XIF. Snitch does not execute the vector or matrix datapath operation. Spatz's decoder interprets the instruction under the current vector and matrix configuration, identifies its execution class, and checks whether the request can be accepted. + +The Spatz controller records the accepted instruction as an operation in progress, associates it with an instruction identifier, and sends the request to the appropriate unit. Matrix configuration instructions are handled in the controller. Ordinary vector arithmetic uses the Vector Arithmetic Unit (VAU), vector memory instructions use the Vector Load/Store Unit (VLSU), and permutation instructions use the Vector Slide Unit (VSLDU). Matrix arithmetic, tile subset moves, tile zero, and context discard use the OPE. Tile subset loads and stores use the VLSU and the OPE's Tile interface. + +Acceptance of a request does not mean that it can begin using operands in the same cycle. The destination unit can buffer a request while an older instruction still owns a source register, destination register, tile, or memory resource. The controller's dependency state and the unit's local ready signals determine when the operation advances. Backpressure can therefore stop a request at dispatch, at an operand port, or at a destination unit without changing the program order observed by software. + +As an operation runs, VRF access and memory responses may proceed over several cycles. A unit eventually returns a completion response with the same instruction identifier. The controller matches that response to the operation in progress, releases its dependency records, and reports completion through XIF when required. This flow applies to both RVV and Zvt operations, even though their results may reside in the VRF, a matrix tile, or TCDM. The precise queues, scoreboards, and completion timing are described in Chapter 4. + +## 3.3 Controller, VRF, and Vector Execution + +The controller maintains RVV configuration such as `vl` and `vtype`, including SEW and LMUL. These fields determine how many elements a vector instruction acts on and which physical registers form each vector register group. The controller also orchestrates the functional units and records register dependencies. In the original Spatz design, progress is tracked at the element level: a consumer can receive ready elements from an older vector producer before that producer has completed its entire vector instruction. Operand backpressure enforces hazards while allowing this vector chaining. [Spatz paper, Section IV-B](https://arxiv.org/html/2309.10137#S4.SS2) + +The centralized VRF holds vector operands and intermediate values for the VAU, VLSU, and VSLDU. It also holds the matrix source operands used by the OPE and provides the endpoint for tile subset moves. The paper describes a small, latch-based VRF rather than a large private register file in each functional unit; the current VLEN = 512 configuration provides 32 architectural vector registers, or 2 KiB of vector state. Sharing this storage enables ordinary vector instructions to prepare matrix inputs and process tile outputs. The physical VRF port arbitration added for the OPE is described in Section 4.2. [Spatz paper, Section IV-C](https://arxiv.org/html/2309.10137#S4.SS3) + +The VAU performs ordinary vector arithmetic using integer and floating-point units. The VSLDU performs vector slides and other element rearrangements. The VLSU executes conventional unit-stride, constant-stride, and indexed memory operations. When floating-point support is enabled, the FPU sequencer also manages scalar floating-point register access and scalar floating-point memory operations. These are parts of the Spatz baseline; the OPE adds matrix arithmetic and distributed tile storage alongside them. [Spatz paper, Sections IV-B–D](https://arxiv.org/html/2309.10137#S4.SS2) + +## 3.4 Double-Bandwidth VLSU and Memory Paths + +The Spatz paper describes a VLSU built from independent 64-bit memory interfaces. Independent ports can issue accesses to different TCDM banks for unit-stride, strided, and indexed patterns without forming one wide memory transaction. Since responses from different ports can arrive out of order, per-port response buffering and ordered VRF commit keep vector loads compatible with element-wise chaining. The narrow ports also use the same TCDM interconnect width as scalar accesses. [Spatz paper, Section IV-D.2](https://arxiv.org/html/2309.10137#S4.SS4.SSS2) + +This study selects the double-bandwidth VLSU with eight 64-bit memory ports and two VRF-side VLSU interfaces. Its aggregate peak memory request width is 512 bits per cycle when all ports accept a request. The corresponding baseline with four ports offers 256 bits per cycle. These widths describe available ports, not guaranteed sustained bandwidth: TCDM bank conflicts, address alignment, response latency, competing requesters, and the access pattern can all reduce the achieved rate. + +The double-bandwidth VLSU serves two data routes. Conventional vector loads and stores exchange data between TCDM and the VRF. `vtle` and `vtse` exchange data between TCDM and the OPE's tile state through a separate Tile interface. A tile store reads a row or column subset from the OPE before generating memory writes; a tile load collects memory data before presenting the subset to the OPE. The VLSU handles addresses, memory requests, and responses; the OPE owns the tile data. Section 4.5 describes the transfer granularity and synchronization with MAC results. + +## 3.5 Zvt Matrix State and Configuration + +[Zvt version 0.4](https://github.com/riscv/riscv-isa-manual/blob/zvt/src/unpriv/zvt.adoc) adds square matrix tiles as architectural state alongside the vector registers. Matrix arithmetic reads its multiplicands from vector register groups and updates a destination tile. SEW is the source element width; TWIDEN is the accumulator widening factor; and TEW = SEW × TWIDEN is the tile element width. The configured dimensions `tm`, `tn`, and `tk` select the active output rows, output columns, and reduction depth. `tn` is tied to the vector length `vl`. + +The tile edge TE is an implementation parameter and a power of two. At TEW = 32, the architectural state comprises four TE × TE tiles, addressed as `mt0`, `mt4`, `mt8`, and `mt12`. Narrower TEW values reinterpret this same storage as more logical tiles. The current implementation uses TE = 16 and a 32-bit physical accumulator element. These numbers describe the current hardware configuration; Zvt software discovers usable dimensions through configuration instructions rather than assuming a fixed TE. + +`msetmtype` establishes `mtype` and `vtype` from scalar registers. `msetmtypei` supplies SEW, TWIDEN, and `altfmt` as instruction fields and initializes the remaining matrix type fields to zero. Both forms set `vl` to zero and derive a matrix LMUL that respects the available vector elements per register, the tile edge, the widening factor, and KMAX. `msettn` then sets `tn` and `vl`; `msettm` and `msettk` set the other active dimensions. These instructions clamp requested sizes to the supported register and tile capacities. KMAX is one for SEW = 32 and two for SEW = 16 in the FP32 and widening FP16-to-FP32 modes considered here. + +A vector configuration instruction such as `vsetvl` invalidates the current matrix type. It does not erase the stored tile contents. Software can therefore switch to a vector configuration for grouped loads or post-processing and later re-establish matrix configuration before another matrix operation. The architectural effect belongs to the Zvt model; the controller's CSR updates and dependency tracking are described in Section 4.3. + +## 3.6 Zvt Operations and Visible Data Paths + +Matrix Arithmetic Instructions update a tile from two vector source groups. In the FP32 mode, `vtfmm` adds an outer product with one reduction element per output element. In widening FP16-to-FP32 mode, the instruction can cover two reduction elements while accumulating in FP32. For a selected output region, the operation has the form + +```text +C[i,j] += sum over k in the active tk range of A[k,i] × B[k,j]. +``` + +Zvt also defines alternative-format floating-point and integer matrix arithmetic instructions. The kernels examined here focus on FP32-to-FP32 and FP16-to-FP32 `vtfmm`. A matrix operation uses tile state for the result, leaving its vector source groups available to be reused by other tile updates. + +`vtzero` establishes a zero value for one tile before accumulation. `vtdiscard` discards the matrix context. These operations have different scopes: a subsequent operation can use a zeroed tile as an accumulator, while discard ends use of the previous tile values. Section 4.4 explains how the OPE represents logical zero and physically clears accumulator storage. + +A Tile Subset Specifier (TSS) selects a tile, a row or column orientation, and an index. `vtmv.v.t` moves a selected tile subset into a vector register group; `vtmv.t.v` moves vector data into a tile subset. The move instructions require SEW = TEW. For an FP16-to-FP32 kernel, software therefore configures a 32-bit move after the matrix arithmetic phase before moving FP32 results into the VRF. `vtle` and `vtse` transfer selected tile rows or columns directly between memory and tile state, using the VLSU rather than the VRF. + +## 3.7 End-to-End Matrix Data Flow + +A tiled GEMM first establishes matrix configuration and places input operands in vector register groups through conventional vector loads. It initializes an output tile with `vtzero` or loads an existing addend with `vtle`. Successive `vtfmm` instructions read the vector operands and accumulate across `K` into that tile. The input vectors can be reused across multiple output tiles; the output partial sums remain in tile state instead of occupying VRF groups for the full reduction. + +After the reduction, the output can follow either of two visible paths. `vtse` writes the tile subset directly to TCDM. If the next stage needs vector arithmetic, `vtmv.v.t` places the selected tile subset in the VRF, where the VAU can apply bias, scaling, activation, or another element-wise operation before an ordinary vector store. The latter path requires additional tile moves and may require vector and matrix configuration changes. The kernel mapping chapter examines how tile layout, register allocation, and instruction order affect these costs. diff --git a/hw/ip/spatz/src/ope/docs/4. Spatz VME Microarchitecture.md b/hw/ip/spatz/src/ope/docs/4. Spatz VME Microarchitecture.md new file mode 100644 index 000000000..a8e3012c8 --- /dev/null +++ b/hw/ip/spatz/src/ope/docs/4. Spatz VME Microarchitecture.md @@ -0,0 +1,219 @@ +# 4. Spatz VME Microarchitecture + +## 4.1 Implementation Overview + +The Spatz VME implementation extends the vector execution subsystem with matrix configuration state and an Outer Product Engine (OPE) that contains a distributed accumulator array and a distributed FPU array. The architectural behavior of the Zvt instructions and matrix tile state is defined in the preceding chapter. This chapter focuses on their hardware implementation and integration with the existing Spatz vector datapath. + +The decoder classifies Zvt instructions according to the instruction categories defined by the specification. Matrix Configuration Instructions, including `msetmtype`, `msetmtypei`, `msettn`, `msettm`, and `msettk`, are executed by the Spatz controller. Matrix Arithmetic Instructions, including `vtfmm`, `vtfmm.alt`, `vtmmu`, and `vtmms`, are dispatched to the OPE. Move Tile Subset instructions (`vtmv.v.t` and `vtmv.t.v`), the Matrix Tile-Zero instruction (`vtzero`), and the Zvt Context Discard instruction (`vtdiscard`) are also handled by the OPE. Load/Store Tile Subset instructions (`vtle` and `vtse`) are dispatched to the VLSU. + +The VAU, OPE, and VLSU have distinct execution roles. The VAU executes conventional RVV arithmetic operations. The OPE performs matrix arithmetic, maintains the accumulator tile state, and moves tile subsets between the distributed accumulator array and the VRF. The VLSU performs vector-memory transfers and moves tile subsets directly between memory and the distributed accumulator array through the Tile interface. + +The controller assigns every issued RVV or Zvt operation an instruction identifier from a common pool of eight identifiers. The identifier remains associated with the operation while it is buffered or executing and is returned by the execution unit with the completion response. The controller uses it to associate an operation with its vector-register dependencies, accumulator-tile dependencies, execution-enable signal, and retirement state. RVV and Zvt instructions can therefore share one dependency domain while executing in the VAU, OPE, or VLSU. + +Request acceptance and execution eligibility are decoupled. An OPE or VLSU queue may accept a request before its recorded dependencies have been resolved. The controller later enables that request through its instruction identifier, and the execution unit combines this enable with its local resource availability before executing the operation. This separation between queue admission and execution eligibility allows the execution units to absorb temporary VRF, arithmetic-pipeline, accumulator, and memory backpressure. + +The distributed accumulator array has three hardware access paths. The FPU array uses its internal read and writeback paths for matrix arithmetic. Tile subset moves use the OPE's VRF-facing path. Tile subset loads and stores use the Tile interface between the VLSU and the accumulator array, without transferring their data through the VRF. + +## 4.2 Shared VRF Access and VAU–OPE Arbitration + +The OPE reuses the existing Spatz VRF for matrix operands and tile subset moves. The VRF provides three 256-bit read ports and one 256-bit write port. The Spatz top module adds three fine-grained round-robin arbitration points that select between VAU and OPE requests when both units request the same port resources in the same cycle. + +The first arbitration point controls the first two VRF read ports as one atomic operand pair. A VAU instruction uses the pair for its vector source operands, while an OPE Matrix Arithmetic Instruction uses it for the left-hand and right-hand matrix operands. The selected requester receives both ports together, preventing the two operands of one operation from being split across different execution units. + +The second arbitration point independently controls the third VRF read port. The VAU uses this port for operations that require a destination-vector read. The OPE uses it for `vtmv.t.v`, which reads a vector-register group and writes the data into a tile subset of the accumulator array. + +The third arbitration point controls the VRF write port. The VAU uses this port for conventional vector-result writeback, while the OPE uses it for `vtmv.v.t`, which writes an accumulator tile subset into a vector-register group. Each arbiter returns a grant only to the selected requester, so a losing request remains asserted and is backpressured until the required port becomes available. + +The VLSU retains separate VRF-side interfaces for conventional vector loads and stores. Load/Store Tile Subset instructions use the independent VLSU–accumulator Tile path and do not request the shared OPE/VAU VRF ports. + +Sharing the existing ports avoids increasing the VRF port count and the associated storage and interconnect cost. The trade-off is increased contention when VAU pre-processing or post-processing overlaps OPE execution. Simultaneous requests cause one unit to wait for the round-robin grant, which can delay operand delivery or result writeback and reduce the amount of useful overlap between vector and matrix execution. The impact therefore depends on the scheduling of vector preparation, matrix arithmetic, and vector post-processing around the shared-port demand. + +## 4.3 Controller and Dependency Management + +The Spatz controller coordinates architectural state and dependencies across the vector and matrix execution units. It maintains the matrix configuration, allocates the common instruction identifiers described in Section 4.1, and records vector-register and accumulator-tile dependencies for in-flight operations. The resulting per-identifier dependency state controls when a queued operation may begin execution. + +### 4.3.1 Matrix Configuration State + +Matrix Configuration Instructions are executed by the controller. The controller maintains `mtype` together with the existing RVV configuration state and derives the active SEW, TEW, TWIDEN, LMUL, `vl`/`tn`, `tm`, and `tk` values used by subsequent Zvt operations. It applies the limits defined by the Zvt configuration rules and by the implemented tile edge, vector length, element widths, and maximum reduction extent. + +The controller also implements the Zvt requirement that every Vector Configuration Instruction invalidates the current matrix type state by clearing `mtype`. Conversely, `msetmtype` and `msetmtypei` update both the matrix and vector configuration fields required for matrix execution. + +Each dispatched Zvt request carries a snapshot of the relevant vector and matrix configuration. A request already buffered in the OPE or VLSU therefore retains the interpretation established when it was dispatched, even if a later configuration instruction changes the controller's architectural state. + +### 4.3.2 Vector Register Scoreboard + +The controller maintains VRF read and write tables indexed by physical vector register. When an instruction is dispatched, these tables are used to construct RAW, WAR, and WAW dependencies between its instruction identifier and older in-flight operations. + +For a vector register group, the scoreboard records every physical vector register selected by LMUL rather than recording only the aligned base register. This preserves dependencies across changes between vector configurations established by `vsetvl` and matrix configurations established by `msetmtype`, even when the two configurations interpret the same architectural register specifier with different group sizes. + +For Matrix Arithmetic Instructions, the scoreboard records both explicitly encoded source register groups and the implicit row-stride groups required by Zvt when `tk` is greater than one. Successive operand rows are separated by `8/KMAX` vector registers, and every LMUL-sized register group belonging to each active row is entered into the read table. A later writer therefore remains ordered after every explicit and implicit source group of the matrix operation. + +### 4.3.3 Matrix Tile Scoreboard + +The controller maintains a separate matrix scoreboard for accesses to accumulator tiles. Each dispatched tile operation creates a claim containing an accumulator-tile mask, read and write attributes, its execution-unit class, and dependencies on older overlapping claims. The mask accounts for the aliasing of logical tile identifiers at different TEW values. + +A dependency is recorded when the new claim overlaps an older accumulator-tile claim and at least one of the two accesses writes the selected state. The controller removes the dependency when the older OPE or VLSU operation returns its completion response. For each instruction identifier, `matrix_enable` is asserted only when its recorded matrix dependencies are empty. + +Every queued OPE and tile-memory request is gated by its `matrix_enable` signal before it becomes eligible for execution. The exception in dependency construction is a Matrix Arithmetic Instruction following another Matrix Arithmetic Instruction: the controller does not create a MAC-to-MAC tile dependency because the OPE can preserve their program order and forward a pipeline-resident partial sum to a subsequent compatible update. Such a MAC still checks `matrix_enable`; it is simply not blocked by older MAC claims. It may remain blocked by an older overlapping move, load, store, zero, or discard claim. + +After a MAC request passes the controller-provided enable, the OPE performs the physical hazard checks required by its implementation. It compares the requested tile and spatial beat with pipeline-resident result tags to decide whether the update can consume a forwarded addend, must wait for an earlier beat, or must switch the resident tile and write older results back. These OPE checks determine when the arithmetic request can execute and complement the architectural ordering enforced by the controller scoreboard. + +## 4.4 Outer Product Engine + +![Spatz VME microarchitecture]() + +*Figure 4.1: Spatz VME microarchitecture and OPE datapath.* + +### 4.4.1 Organization + +The OPE contains a distributed accumulator array, a distributed FPU array, and control logic for request buffering, spatial scheduling, result forwarding, accumulator writeback, and tile subset moves. The two arrays are spatially aligned: every FPU lane is paired with one lane-local accumulator bank. In the current implementation, the compute edge is CE = 8, producing an 8 × 8 compute array with 64 FPU lanes and 64 accumulator banks. + +The accumulator banks hold materialized tile state. The FPU pipeline can temporarily hold a newer version of the elements currently being updated. This distinction permits successive Matrix Arithmetic Instructions to forward partial sums through the FPU pipeline instead of writing and rereading the accumulator after every instruction. + +### 4.4.2 Request Queues and Operation Selection + +The OPE separates incoming requests into four operation classes. Matrix Arithmetic Instructions enter a four-entry, non-fall-through FIFO. Tile-to-vector moves, vector-to-tile moves, and tile-clean operations each use a dedicated spill-register queue. + +A Matrix Arithmetic Instruction can occupy the OPE for more than ten cycles because its vector groups, reduction rows, and spatial beats are processed over multiple cycles. The four-entry MAC FIFO allows several later arithmetic instructions to wait close to the compute datapath. Buffered work can therefore remain available when Snitch stalls or does not immediately supply another MAC, reducing gaps between long-running Matrix Arithmetic Instructions and helping keep the FPU pipeline occupied. + +The MAC FIFO is non-fall-through so a newly accepted request is registered before it is considered for execution. On the following cycle, the controller's registered dependency state and the request's `matrix_enable` value are available together. This avoids a combinational path in which a request could pass directly from dispatch into the MAC scheduler before its dependencies have been recorded. + +The four queue heads are considered using a fixed priority: tile-clean, tile-to-vector, vector-to-tile, and finally MAC. A queue head participates only after its `matrix_enable` signal is asserted. The selected operation must also find the resource used by its class available: a move requires its move datapath and accumulator access to be idle, while a MAC requires space in the MAC scheduler and no conflicting tile-interface access or resident-tile drain. + +If a queue head is not selected, its valid bit and request payload remain in that queue. It is reconsidered in a later cycle after the higher-priority operation advances or the blocking resource becomes available. Backpressure reaches the controller only when the corresponding queue can no longer accept another request. + +### 4.4.3 Distributed Accumulator Array + +The accumulator array stores tile state across CE² lane-local banks. Each bank uses 32-bit physical words, matching the default TEW = 32 accumulation configuration. When TEW is narrower, such as TEW = 16 or TEW = 8, a logical tile selects a half-word or byte slice of the same physical word. Byte-enable signals allow one slice to be updated while preserving the neighbouring logical tiles that share the word. + +Zvt constrains the tile edge TE to a power of two. The implementation may choose a TE larger than the physical compute edge CE, in which case a logical TE × TE tile is divided into CE × CE spatial beats. The number of beats per physical tile slot is + +```text +SpatialBeats = (TE / CE)². +``` + +Each bank contains four TEW = 32 physical tile slots, so its accumulator depth is + +```text +AccDepth = 4 × SpatialBeats. +``` + +For TE = 16 and CE = 8, one tile contains four spatial beats and every bank contains sixteen 32-bit words. The complete distributed accumulator has a capacity of 64 × 16 × 32 bits, or 4 KiB. + +For an element at logical coordinates `(row, col)`, `row mod CE` and `col mod CE` select the lane-local accumulator bank. The upper coordinate bits select the spatial beat: + +```text +beat_row = floor(row / CE) +beat_column = floor(col / CE) +spatial_beat = beat_row × (TE / CE) + beat_column. +``` + +The logical tile identifier selects the physical tile slot and the active byte range within its 32-bit word. The physical address within a lane-local bank combines the physical tile slot with the spatial-beat index. + +### 4.4.4 Distributed FPU Array + +The CE × CE FPU array performs one outer-product update for the spatial region selected by the scheduler. A CE-element segment from the left operand is distributed across the compute rows, and a CE-element segment from the right operand is distributed across the compute columns. The FPU lane at `(r, c)` updates the accumulator element associated with that compute-array position. + +The FP32 path uses one FMA datapath per lane. The widening FP16-to-FP32 path uses an SDOTP datapath that multiplies two pairs of FP16 subwords and adds their two products to one FP32 partial sum. Both datapaths share the same accumulator addressing, active-lane mask, pipeline tags, forwarding path, and writeback control. + +The active dimensions `tm` and `tn` may end within a spatial beat. The scheduler derives a CE × CE lane mask from the beat coordinates and suppresses lanes outside the active region. Suppressed lanes neither update the FPU pipeline state nor write the accumulator banks. + +### 4.4.5 Spatial Scheduling + +The MAC scheduler visits the active spatial beats of a tile in row-major beat order. With TE = 16 and CE = 8, a full tile is divided into four 8 × 8 regions. These regions form four independent accumulation chains through the four-stage FPU pipeline. After all four beats have issued, the result of the first beat has reached the forwarding point when the scheduler returns to that beat for the next reduction update. + +Two consecutive MAC operations are dependent when they update the same tile and the same spatial beat. Their destination tiles are compatible when they identify the same tile. A dependent MAC waits until the preceding result reaches the forwarding point and then consumes that result as its addend. Consecutive updates to different beats of a compatible tile can occupy different FPU pipeline stages at the same time. + +A boundary tile with a smaller `tm` or `tn` may have fewer active spatial beats. When the number of active beats is smaller than the FPU latency, the scheduler can return to a dependent beat before its preceding result reaches the forwarding point. The scheduler keeps the MAC active and issues that beat when the required result becomes available. + +The scheduler preserves MAC program order. It holds one current operation and one look-ahead operation, allowing the next queued MAC to become active as the current instruction issues its final beat. + +### 4.4.6 Pipeline Residency, Forwarding, and Drain + +Pipeline residency is the period during which the newest results of a MAC remain in the FPU pipeline instead of being stored in the accumulator banks. Each pipeline stage carrying an unfinished MAC result also carries a tag with the logical tile identifier, TEW, spatial beat, active-lane mask, and writeback state. The resident tile is the tile identified by these tagged results. The term describes the tile currently retained in the FPU pipeline and does not restrict the number of tiles stored in the distributed accumulator array. + +When a dependent MAC reaches the scheduler, the result for its compatible tile and matching spatial beat is forwarded directly from the final FPU pipeline stage as the next addend. The pipeline therefore preserves the newest value while the corresponding accumulator location can still contain an older value. The forwarded MAC result replaces that older accumulator value in the arithmetic dependency chain. + +When a MAC targets a different tile, the destination is incompatible with the resident tile. The OPE marks the resident results for writeback, drains them into the accumulator banks, and then changes the resident tile. A mismatch between the next requested spatial beat and the result at the forwarding point also causes the older result to advance to accumulator writeback before the conflicting beat proceeds. + +A tile subset move or a VLSU tile access is an external operation because it accesses a tile outside the FPU pipeline through either the VRF-facing move path or the VLSU Tile interface. Before an external operation reads or writes the resident tile, the OPE drains its newest MAC results into the accumulator banks. The external operation remains stalled until the drain finishes, so it observes or replaces the current tile value. + +### 4.4.7 Tile Subset Moves + +The tile-to-vector and vector-to-tile movers are fully pipelined at the VRF-word level. Once a move is active and the required VRF port is granted, the mover transfers one 256-bit VRF word per cycle. The last word of one move can retire while the next queued move is accepted, allowing back-to-back moves without an idle cycle between them. + +A tile-to-vector move reads the selected tile rows or columns from the accumulator banks, selects the configured element slice from each 32-bit physical accumulator word, and packs the elements into consecutive 256-bit VRF words. Logical-zero metadata overrides stale physical data with zero before packing. A vector-to-tile move performs the reverse mapping. It captures an accepted VRF word until the accumulator write succeeds and updates only the selected bytes of each accumulator word. Row and column orientations use different bank and spatial-beat traversal patterns while sharing the same VRF datapath. + +Zvt requires SEW to equal TEW for a Move Tile Subset instruction. The current hardware configuration uses 32-bit accumulator elements and TEW = 32 for tile moves. With TE = 16, one tile row contains 16 × 32 = 512 bits, so either move direction transfers that row as two 256-bit VRF words over two cycles. The accumulator datapath supports byte and half-word selection for TEW = 8 and TEW = 16 configurations, but those modes require a corresponding hardware TEW configuration. + +### 4.4.8 Logical Tile Zero + +`vtzero` establishes a logical zero value for the selected tile without immediately clearing its physical accumulator words. Before accepting `vtzero`, the OPE waits until no queued or executing MAC is using the tile. If the tile is resident in the FPU pipeline, the OPE drains its tagged results without writing them into the accumulator banks because the zero operation supersedes those results. + +When `vtzero` is accepted, the OPE sets logical-zero metadata for the selected element slice in every spatial beat of the tile. A later MAC targeting a marked slice receives zero as its FPU addend. Tile-to-vector and VLSU reads also return zero for that slice even though the accumulator banks can still contain an older physical value. + +A MAC writeback, vector-to-tile move, or VLSU tile write clears the logical-zero metadata for each byte slice it replaces. The newly written physical value is then visible to subsequent operations. `vtzero` does not issue the accumulator-wide physical flush signal. + +### 4.4.9 Context Discard + +`vtdiscard` invalidates the complete matrix accumulator context. The OPE accepts it only after MAC scheduling, tile moves, and Tile-interface accesses have become idle. If MAC results remain resident in the FPU pipeline, the OPE first drains and invalidates those results without writing them into the accumulator banks. + +After the pipeline is empty and no accumulator access is active, `vtdiscard` asserts the accumulator flush signal. Every lane-local accumulator bank clears its physical storage on that clock edge, and the OPE clears all logical-zero metadata. Hardware reset performs the same physical clearing of the accumulator banks. + +### 4.4.10 MAC Completion + +A MAC produces its local completion event when its final required reduction and spatial beat is accepted by the FPU pipeline. The result of that beat may still be travelling through the pipeline when the instruction identifier is returned to the controller. Its pipeline tag continues to protect the destination tile after instruction completion. + +A dependent MAC can consume a result belonging to a compatible resident tile through forwarding. An incompatible MAC causes the resident results to be written to the accumulator before changing the resident tile. An external operation also triggers writeback before accessing a resident tile. MAC completion marks the end of operand issue for that instruction and does not require its results to have reached the accumulator banks. + +## 4.5 Vector Load Store Unit Tile Path + +### 4.5.1 Request Buffering and Execution + +The double-bandwidth VLSU executes conventional vector-memory operations and Load/Store Tile Subset Instructions. Incoming LSU requests enter a four-entry, non-fall-through FIFO in program order. A tile request remains at the queue head until its `matrix_enable` signal is asserted, its Tile Subset Specifier is valid, and the tile-transfer state machine is available. + +When the tile-transfer state machine accepts a request, it records the tile identifier, row or column orientation, starting index, active element count, TEW, memory address, and instruction identifier. Conventional vector-memory requests continue through the VRF-facing datapath. A tile request uses the independent Tile interface between the VLSU and the OPE accumulator array. + +The VLSU performs address generation and memory transfer for the request. The OPE decides when the requested accumulator tile can be accessed. A request that cannot yet access the Tile interface remains active in the VLSU and holds its request payload stable until the OPE asserts ready. + +### 4.5.2 Tile Interface and Data Mapping + +The Tile interface provides separate read and write channels between the VLSU and the distributed accumulator array. Load/Store Tile Subset Instructions therefore do not use the shared OPE/VAU VRF ports. The physical Tile word width is fixed when the hardware is generated: + +```text +TileWordWidth = TE × AccumulatorElementWidth. +``` + +With TE = 16 and a 32-bit physical accumulator element, one Tile word is 512 bits. TEW selects the byte, half-word, or word slice that represents each architectural tile element within the 32-bit accumulator words. + +A row-oriented transfer selects consecutive columns from one tile row. A column-oriented transfer selects one column across consecutive tile rows. For `vtse`, the OPE gathers the selected elements into a packed Tile word, and the VLSU writes that payload to memory. For `vtle`, the VLSU receives a packed payload from memory, and the OPE scatters its elements into the selected accumulator locations. + +A `vtle` transfer updates only the byte slices selected by TEW. The OPE preserves the other slices in each physical accumulator word through a read-modify-write operation. Row and column transfers use different bank and spatial-beat traversal patterns but share the same Tile interface. + +### 4.5.3 Logical-Zero Handling + +The OPE applies logical-zero metadata when the VLSU reads a tile for `vtse`. The accumulator banks can still contain an older physical value after `vtzero`, so the OPE reads the physical word and replaces every marked byte slice with zero before constructing the Tile word. Unmarked slices retain their physical accumulator values. Memory therefore receives the architectural tile value even when only part of a 32-bit accumulator word is logically zero. + +For each selected slice, the value returned through the Tile read channel is equivalent to: + +```text +TileReadValue = LogicalZero ? 0 : PhysicalAccumulatorValue. +``` + +A `vtle` write replaces the selected physical accumulator slices and clears their logical-zero metadata. Subsequent MAC, move, and VLSU reads observe the newly loaded values. Logical-zero metadata for slices outside the `vtle` write remains unchanged. + +### 4.5.4 TCDM Transfers + +The VLSU divides a packed Tile word into ELEN-wide TCDM requests and distributes consecutive chunks across the available memory ports. In the current double-bandwidth configuration, eight 64-bit ports provide an aggregate peak transfer width of 512 bits per cycle. A complete 512-bit Tile word can be presented in one request wave when all eight accesses are accepted together. + +The sustained transfer rate depends on the number of active elements, address alignment, TCDM bank availability, outstanding responses, and the gather or scatter pattern required by the selected orientation. Per-port tags distinguish tile traffic from conventional vector-memory traffic and record each response position within the packed Tile word. + +For `vtle`, the VLSU collects the memory responses before presenting the completed Tile word to the OPE. The instruction completes after the OPE accepts the Tile write. For `vtse`, the VLSU first obtains the Tile word from the OPE, issues all required memory writes, and waits for their acknowledgements before reporting completion. + +### 4.5.5 Coordination with Pipeline-Resident Results + +A Load/Store Tile Subset Instruction is an external operation because it accesses a tile through the Tile interface rather than through the FPU pipeline. The accumulator banks must contain the newest tile value before `vtse` reads that tile. The OPE therefore detects a matching resident tile, writes its tagged MAC results into the accumulator banks, and withholds Tile-read ready until the drain finishes. + +Before `vtle` writes a tile, the OPE checks for queued, executing, or pipeline-resident MAC updates to the same tile. A matching resident tile is drained before the Tile write is accepted. The controller's matrix scoreboard orders the `vtle` request against older matrix operations, while the OPE prevents a physical accumulator access from overlapping its local MAC or move activity. + +The VLSU holds the Tile request stable during either wait. After the Tile-interface handshake and all required memory activity complete, the VLSU returns the request's instruction identifier to the controller. The controller then retires the corresponding matrix-scoreboard claim. diff --git a/hw/ip/spatz/src/ope/docs/5. System-Level and ISA Exploration.md b/hw/ip/spatz/src/ope/docs/5. System-Level and ISA Exploration.md new file mode 100644 index 000000000..2831fb5be --- /dev/null +++ b/hw/ip/spatz/src/ope/docs/5. System-Level and ISA Exploration.md @@ -0,0 +1,63 @@ +# 5. System-Level and ISA Exploration + +Chapter 4 described the implemented controller, OPE, and VLSU. This chapter examines what their parameters and the Zvt instruction model imply for a matrix workload before choosing a kernel schedule. It separates architectural instruction demand from physical OPE work, accumulator capacity, and movement across the TCDM, VRF, and Tile interface. Chapter 6 will address register allocation, output layout, and instruction scheduling. + +## 5.1 Parameters and Scope + +The application dimensions are `M`, `N`, and `K` for `C[M,N] += A[M,K] × B[K,N]`. The relevant architecture parameters are the vector length `VLEN`, the matrix tile edge `TE`, the physical OPE compute edge `CE`, source element width `SEW`, tile element width `TEW`, and the maximum reduction extent `KMAX`. The active reduction extent `tk` cannot exceed `KMAX`; the active output dimensions `tm` and `tn` cannot exceed the configured tile and vector-group limits. + +The present Spatz configuration has VLEN = 512 bits, four FPUs, an eight-port double-bandwidth VLSU, and a 128 KiB TCDM divided into 16 banks. The OPE has CE = 8, TE = 16, and 32-bit physical accumulator elements. FP32-to-FP32 and widening FP16-to-FP32 are the arithmetic modes considered in this study. Other parameter values below are analytical design points; a result obtained for one of them is not a measurement of the present RTL. The [Spatz paper](https://arxiv.org/html/2309.10137#S3) motivates the compact 2 KiB VRF used at VLEN = 512, while [Zvt version 0.4](https://github.com/riscv/riscv-isa-manual/blob/zvt/src/unpriv/zvt.adoc) defines the tile and configuration constraints. + +## 5.2 Vector Capacity and Matrix Configuration + +Spatz has 32 architectural vector registers. At VLEN = 512, their total capacity is `32 × 512 / 8 = 2,048 bytes`. One register contains `EVE = VLEN / SEW` source elements: 16 FP32 elements or 32 FP16 elements. Zvt derives the matrix LMUL from the effective tile edge `ETE`, EVE, KMAX, and the widening factor TWIDEN. For the modes considered here, TEW = 32 and hence ETE = TE. + +```text +LMUL = min(ceil(ETE / EVE), 8 / KMAX, 8 / TWIDEN) +``` + +For TE = 16, both FP32-to-FP32 (`EVE = 16`, `KMAX = 1`, `TWIDEN = 1`) and FP16-to-FP32 (`EVE = 32`, `KMAX = 2`, `TWIDEN = 2`) select LMUL = 1. At the analytical TE = 32 point, FP32 selects LMUL = 2 while widening FP16 still selects LMUL = 1. Thus a larger TE can reduce the number of matrix instructions while increasing the vector-register group required by an operand. The configuration instructions clamp `tm`, `tn`, and `tk` to the resulting limits. No kernel may assume that its requested dimensions were accepted without checking the values returned by those instructions. + +VLEN is held at 512 throughout the evaluation. Sweeping it would change VRF capacity, vector instruction coverage, and possibly matrix LMUL at the same time. The comparisons in this chapter therefore isolate other parameters instead of attributing a change caused by several coupled resources to VLEN alone. + +## 5.3 Architectural Instructions and Physical Compute Work + +For a full tile, one Matrix Arithmetic Instruction covers a `TE × TE` output region and `tk` positions of the reduction dimension. If `M`, `N`, and `K` divide evenly by TE and `tk`, the instruction count and physical spatial-beat count are + +```text +N_matrix = (M / TE) × (N / TE) × (K / tk) +N_beats = N_matrix × (TE / CE)² + = (M × N × K) / (CE² × tk) +``` + +For boundary tiles or a final short reduction group, `ceil(M / TE) × ceil(N / TE) × ceil(K / tk)` gives the number of matrix instructions when each tile and reduction group is configured separately. The actual work in the final tile or group is smaller than the full-tile upper bound; its active dimensions are set through `tm`, `tn`, and `tk`. The arithmetic problem still contains `M × N × K` scalar multiply-accumulate contributions regardless of the chosen instruction grouping. + +For example, take `M = N = K = 64`, TE = 16, and CE = 8. FP32-to-FP32 uses `tk = 1`, giving 1,024 matrix instructions and 4,096 spatial beats. Widening FP16-to-FP32 can use `tk = 2`, giving 512 matrix instructions and 2,048 spatial beats. Both modes update the same number of output elements across the full reduction. The widening datapath performs two FP16 products per FP32 accumulation step, so beat counts alone do not express equivalent FPU operations or execution cycles. + +At the analytical TE = 32, CE = 8 point, the same FP32 workload needs 256 matrix instructions with 16 spatial beats each, again totaling 4,096 beats. Changing TE changes how much work a single instruction describes; changing CE changes the number of physical regions through which that instruction is executed. The latter also changes compute-array size, so the beat count cannot by itself predict area, utilization, or runtime. + +## 5.4 Accumulator and Transfer Scaling + +At TEW = 32, Zvt provides four independently addressable tiles. Ignoring control metadata, their architectural storage scales as + +```text +TileStateBytes = 4 × TE² × (TEW / 8) +``` + +The present TE = 16 configuration therefore provides `4 × 16² × 4 = 4 KiB` of tile storage. A TE = 32 design point would require 16 KiB for the same four TEW = 32 tiles. This fourfold growth follows from doubling both tile dimensions; it is not a measured area or energy estimate. Narrower TEW values reinterpret the tile storage as more logical tiles rather than increasing the total bytes. + +The physical Tile interface width also depends on TE: `TileWordWidth = TE × 32 bits` in the present 32-bit accumulator organization. A TE = 16 row is 512 bits, matching the aggregate peak width of eight 64-bit TCDM ports. A TE = 32 row would be 1,024 bits and require at least two full-width memory request waves even under ideal acceptance. Alignment, TCDM bank conflicts, and response timing can add cycles. These widths concern the VLSU-to-tile path; a tile subset moved to the VRF uses 256-bit VRF words and a separate path. + +Each input or output must be counted at the boundary it crosses. A conventional vector load moves an operand from TCDM to the VRF; a matrix instruction reads that operand from the VRF into the OPE. Reusing a loaded register avoids another TCDM load, although it does not eliminate each OPE operand access. A tile store moves a result from the OPE to TCDM without using the VRF. A tile-to-vector move followed by a vector store adds a Tile-to-VRF transfer and a VRF-to-TCDM transfer, but makes the result available to VAU post-processing. Element counts, transferred bytes, instruction counts, and transfer cycles are therefore distinct quantities. + +## 5.5 System-Level Limits to Examine + +Three possible limits follow from the architecture. First, Snitch must supply matrix configuration, vector loads, matrix arithmetic, address updates, and output instructions. A larger TE reduces arithmetic instruction demand for a fixed problem, but the front end still has to supply the other instructions. Second, the OPE needs operands from the shared VRF while the VAU may be preparing or processing vectors; contention can delay an otherwise ready operation. Third, tile stores and moves must observe completed tile values, so a result still resident in the FPU pipeline may have to reach the accumulator before the transfer proceeds. + +The double-bandwidth VLSU raises the available peak TCDM request width, but sustained traffic also depends on banks, addresses, and concurrent requesters. Increasing TE raises tile capacity and Tile interface width as well as the work per matrix instruction. Changing precision alters input bytes, KMAX, and the available arithmetic datapath. These effects must be considered together when selecting an experiment. An instruction-count reduction alone does not establish a faster kernel. + +## 5.6 Exploration Boundary + +The supported reference point for the following chapters is VLEN = 512, CE = 8, TE = 16, with FP32-to-FP32 and widening FP16-to-FP32 arithmetic. TE = 32 and other parameter values provide analytical comparisons of instruction density, tile capacity, and transfer width. Measured latency, port contention, and utilization require an implemented configuration and an emitted instruction stream. + +This chapter identifies the resources a kernel must manage. Chapter 6 will specify how operands, output tiles, vector registers, and stores are arranged over time. Later measurements can then compare a concrete instruction sequence with the work and traffic predicted here. diff --git a/hw/ip/spatz/src/ope/docs/6. Kernel Design.md b/hw/ip/spatz/src/ope/docs/6. Kernel Design.md new file mode 100644 index 000000000..484ae4e1a --- /dev/null +++ b/hw/ip/spatz/src/ope/docs/6. Kernel Design.md @@ -0,0 +1 @@ +# 6. Kernel Design diff --git a/hw/ip/spatz/src/ope/docs/image/VME microarchitecture.pdf b/hw/ip/spatz/src/ope/docs/image/VME microarchitecture.pdf new file mode 100644 index 000000000..10ca59c91 Binary files /dev/null and b/hw/ip/spatz/src/ope/docs/image/VME microarchitecture.pdf differ diff --git a/hw/ip/spatz/src/ope/docs/image/VME microarchitecture.png b/hw/ip/spatz/src/ope/docs/image/VME microarchitecture.png new file mode 100644 index 000000000..0f161c17e Binary files /dev/null and b/hw/ip/spatz/src/ope/docs/image/VME microarchitecture.png differ diff --git a/hw/ip/spatz/src/ope/readme.md b/hw/ip/spatz/src/ope/readme.md index 2dcc61fe6..52b29f5e2 100644 --- a/hw/ip/spatz/src/ope/readme.md +++ b/hw/ip/spatz/src/ope/readme.md @@ -1,7 +1,13 @@ # Spatz OPE Architecture and Dataflow This document describes the current Outer Product Engine (OPE) RTL. It is not an ISA specification or a simulation sign-off. -The top-level module is `spatz_ope.sv`; computation and storage are implemented by `opope_fma.sv` and `opope_accumulator.sv`. +The OPE supports the RISC-V Vector Matrix Extension, excluding memory instructions and matrix-configuration instructions. +Its datapath contains a distributed `CE×CE` MAC array and a distributed `CE×CE` accumulator array. The floating-point +arithmetic is collectively referred to as the FPU and contains both FMA and SDOTP datapaths; the accumulator stores their +results and provides the feedback addend for subsequent operations. + +The top-level module is `spatz_ope.sv`; floating-point arithmetic and storage are implemented by the FPU datapaths and +`opope_accumulator.sv`. Shared request and tile types come from `spatz_pkg` and `rvv_pkg`; floating-point formats come from `fpnew_pkg`. ## 1. Architecture @@ -18,7 +24,7 @@ Decoder + Controller | +--> MAC FIFO --> current/next --> VRF read [0:1] | | | | | | | v | - | | CE x CE FMA pipeline | + | | CE x CE FPU pipeline | | | ^ | | | | +-- forward-+ | | | ^ | drain/write | @@ -32,7 +38,7 @@ Decoder + Controller v L1 TCDM -FMA pipeline <--> FMA-side accumulator port +FPU pipeline <--> FPU-side accumulator port MAC/VT/TV/Clean completion --> response buffers --> arbiter --> Controller VLSU completion --------------------------------------------> Controller @@ -63,17 +69,19 @@ Internal request selection prioritizes Clean, VT, TV, then MAC; active datapaths | Parameter | Value | Meaning | | --- | --- | --- | -| `TE = TileEdge` | 16 | Maximum logical tile edge | +| `TE = TileEdge` | 16 | Maximum logical tile edge in the current configuration | | `CE = OPEComputeEdge` | 8 | Outer-product compute edge per beat | +| `EnableTEW16` | 0 | Elaborate the same-format FP16 FMA datapath when set | | `AccElemWidth` | 32 bits | Fixed physical accumulator word width | | `VRFWordWidth` | 256 bits | Width of each VRF read/write port | -| `NumPipeRegs` | 4 | FMA pipeline register stages | +| `NumPipeRegs` | 4 | Arithmetic pipeline register stages | | `SpatialBeats = (TE / CE)^2` | 4 | Spatial beats for a full tile | | `AccDepth` | 16 | Accumulator entries per compute lane | -| `TileDataWidth` | 512 bits | OPE/VLSU tile data interface width | +| `TileDataWidth` | 512 bits | OPE/VLSU tile-row data width | -Each of the CE x CE lanes has dedicated FP16 and FP32 FMA units and a 32-bit accumulator bank. -Runtime TEW selects the arithmetic path; the two formats do not simultaneously compute two results for one element. +Each CE x CE lane always has an FP32 FMA and two mixed-format SDOTP wrappers for standard FP16 and FP16ALT. +The same-format FP16 FMA is generated only when `EnableTEW16=1`; it is absent by default. +Runtime SEW, TEW and ALT-format state select exactly one result path for an operation. Physical storage is `CE * CE * AccDepth * AccElemWidth / 8`, currently 4096 bytes. TEW8 is supported for byte storage, moves and clearing only. No FP8 MAC is enabled. @@ -95,7 +103,7 @@ C[i,j] += sum(A[i,k] * B[k,j]), i < tm, j < tn, k < tk 5. The lane mask prevents writes outside tm/tn. Issuing the final beat produces MAC completion. The work per instruction is `ceil(tm / CE) * ceil(tn / CE) * tk` beats. -For tm=tn=16 and CE=8, tk=1 uses four beats and tk=2 uses eight. This is issue work, not end-to-end latency under stalls. +For tm=tn=32 and CE=8, tk=1 uses 16 beats and tk=2 uses 32. This is issue work, not end-to-end latency under stalls. **Multiple-beat benefit:** a CE x CE array is reused across a larger TE x TE tile and across K reduction steps. The hardware does not need one FMA per element of the entire tile, and software does not issue a separate instruction for every rank-1 step. @@ -105,24 +113,47 @@ Fringe tiles do not require padded arithmetic to protect the result, and inactiv This avoids valid work on inactive lanes but does not imply every combinational node is clock-gated. Real power and utilization still depend on operand delivery, stores and pipeline stalls. -Each source supplies one VRF word per beat. The configuration must satisfy `CE * SEW <= VRFWordWidth`. +Each source supplies one VRF word per beat. The configuration must satisfy +`CE * SEW <= VRFWordWidth` for FMA and `CE * TEW <= VRFWordWidth` for packed SDOTP. Increasing CE alone does not add a multiword operand collector. ### Different SEW and TEW -SEW controls vector operand extraction, bit offsets and K-row stride. TEW controls accumulation precision and the tile byte footprint. +For same-width FMA, SEW controls operand extraction and K-row stride. For widening +SDOTP, each base-vector element contains packed K lanes and has width TEW. +TEW also controls accumulation precision and the tile byte footprint. `mtype.mtwiden` encodes a multiplier: 01 means 1, 10 means 2, and 11 means 4; 00 invalidates matrix state. The element width is `TEW = SEW * multiplier`, not SEW times the raw encoded field. | SEW / TEW | Current behavior | | --- | --- | | 8 / 8 | Raw-byte tile load/store, TV/VT and VTZERO; no FP8 MAC | -| 16 / 16 | FP16 inputs, FP16 accumulation/rounding, selected 16-bit slice | -| 16 / 32 | FP16 or alternate FP16 inputs widened to FP32, FP32 FMA/accumulation, full word | +| 16 / 16 | Same-format FP16 FMA when `EnableTEW16=1`; unsupported by the default OPE instance | +| 16 / 32 | FP16 or FP16ALT source and FP32 destination through the corresponding SDOTP wrapper | | 32 / 32 | Native FP32 inputs and accumulation | -For SEW16/TEW32, vector loads provide 16-bit operands, and the FP32 FMA input path converts them to FP32 before arithmetic. -The accumulator remains FP32; it does not narrow each result back to FP16. Tile memory transfers use the 32-bit TEW layout. +An `opope_fma` accepts operands only in its configured `FpFormat`; it performs no FP16-to-FP32 type conversion. +When source and destination formats differ, the OPE enables either the FP16-to-FP32 or FP16ALT-to-FP32 +`opope_sdotp_wrapper`. The FP32 FMA is enabled only for a native FP32 source and FP32 destination. +The mixed-format result and accumulator remain FP32. Tile memory transfers use the 32-bit TEW layout. +For FP16-to-FP32 with TK=2, each input word is `{even_K_fp16, odd_K_fp16}`. +OPE maps the high halfword to lane 0 and the low halfword to lane 1, computing +`C += A_even * B_even + A_odd * B_odd`. No implicit-register operand fetch or +operand stream-register buffer is needed. A/B each supply one 256-bit VRF word +per accepted beat (eight packed elements). A 16x16 tile on 8x8 CEs takes four +spatial issue beats, independent of TK; pipeline latency and stalls are additional. +The controller still tracks implicit row-stride register dependencies. +The packed TE16 software keeps matrix arithmetic configured as +SEW16/TWIDEN2/TK2. Store-afterK and store-interleave execute one `msetmtype` +followed by `msettn 16` and then interleave individual VLE32 operations with +VTFMM instructions. VLE32 uses its instruction EEW for the memory transfer, so +it transfers sixteen packed 32-bit elements while matrix VSEW remains 16. + +The vsetvl variant deliberately groups four packed rows in one vector load. +It executes `vsetvli e32,m4`, which produces VL=64 and lets one VLE32 fill +v0-v3, v8-v11, v16-v19 or v24-v27. `msetmtype` then restores the unchanged +SEW16/TWIDEN2/TK2 state. The following `msettn` receives 64, but the controller +clamps TN and VL to the configured ETE=16. TV/VT are bit transfers, not floating-point conversions, and require SEW==TEW. To move a FP32 tile through the VRF, establish valid SEW32/TEW32 matrix state; `vsetvli` alone is insufficient because it clears mtype. Complete old-format work appropriately before changing configuration; do not reinterpret slices while old results remain resident. @@ -168,13 +199,14 @@ and resetting zero metadata. Reset also physically clears memory. When draining for VTZERO or discard, superseded resident results can be consumed without writing them back. VTZERO initializes one logical tile; VTDISCARD clears all tile state and must not be used as a single-byte-tile clear. -## 3. FMA Pipeline, Forwarding and Switching +## 3. Arithmetic Pipeline, Forwarding and Switching ### Data and Tags -Each `opope_fma` has four distributed register stages. The FP16/FP32 ready signals are aggregated into `fma_pipe_ready`. -`fma_pipe_advance` advances computation and tag state together. Tags contain valid, tile ID, TEW, spatial beat, lane mask and `write_acc`. -FMA results do not need an instruction ID because MAC completion is generated at issue. Request and response queues still retain IDs +The FMA and SDOTP datapaths have four distributed register stages. Their ready signals are aggregated into `fma_pipe_ready`; +disabled or unselected paths remain ready and do not produce a valid result. +`fma_pipe_advance` advances the selected computation, its valid pipeline and tag state together. Tags contain valid, tile ID, +TEW, spatial beat, lane mask and `write_acc`. Arithmetic results do not need an instruction ID because MAC completion is generated at issue. Request and response queues still retain IDs because one core can have multiple outstanding instructions. Each FMA tag stage includes a valid bit. Pipeline empty/full uses OR/AND reductions of the valid vector, while @@ -196,7 +228,7 @@ Forwarding uses actual tags and handshakes, not a fixed software instruction-spa A different incoming MAC tile raises `resident_switch_req`; the switch occurs when that MAC actually fires. Existing pipeline tags are marked `write_acc`, so old-tile results write back while the new tile starts from its own accumulator data. -Overlap is permitted only when operand, busy and FMA-ready conditions allow it. +Overlap is permitted only when operand, busy and FPU-ready conditions allow it. External VTSE/VTLE, VT/TV or clean operations that need resident state trigger a drain. Drain completion uses the old tile's next-state busy map, not global pipeline emptiness; successor work may remain in flight. @@ -205,7 +237,7 @@ supersede them and can discard the drained values. A full drain-reason enum is t An external same-tile read can trigger draining even with a younger MAC queued. That younger MAC waits for the read. Requiring every queued MAC to disappear first could create a circular wait between VTSE and MAC. -## 4. Accumulator Byte Mapping +## 4. Accumulator Organization and Byte Mapping A logical tile ID names an architectural view. A physical slot stores 32-bit words. TEW16 places two logical tiles in a slot; TEW8 places four. These are views of the same storage, not additional copies. @@ -237,25 +269,73 @@ one muxed port: | Side | Read purpose | Write purpose | Transfer organization | | --- | --- | --- | --- | -| FMA | Read one selected lane/beat as the addend | Write one retired FMA result | One 32-bit word per lane | +| FPU | Read one selected lane/beat as the addend | Write one retired FMA or SDOTP result | One 32-bit word per lane | | VRF | VT tile-to-vector read and TV read-modify-write | TV vector-to-tile update | Grouped words feeding a `VRFWordWidth` move | | VLSU | VTSE tile-row read and VTLE read-modify-write | VTLE memory-to-tile update | `TileDataWidth`, currently 512 bits | -The FMA side uses `fma_raddr_i/fma_rdata_o` and `fma_waddr_i/fma_wdata_i/fma_wen_i`. -The VRF and VLSU sides each have independent grouped read and write addresses, data and byte enables. -Consequently, accesses to non-conflicting accumulator locations can occur in the same cycle: for example, an FMA result may -write one tile while TV updates another and VTSE reads a third. +The FPU side uses `fma_raddr_i/fma_rdata_o` and `fma_waddr_i/fma_wdata_i/fma_wen_i`. +For a MAC operation, the FPU read address is generated from the selected physical tile slot and spatial beat. The selected +32-bit word is then used as the addend; TEW8 selects one byte, TEW16 selects one half-word, and TEW32 selects the complete +word. When the arithmetic pipeline retires, the result is written back to the same physical slot and beat with the matching +byte enables. Thus, the accumulator memory stores the feedback value, while the FPU performs the actual multiply-accumulate +operation through either the FMA or SDOTP datapath. + +The physical address is generated in two stages. First, the architectural tile ID is converted into a physical slot and a +byte or half-word offset. For example, `mt0`, `mt4`, `mt8`, and `mt12` select physical slots 0, 1, 2, and 3 for TEW32. +For a matrix element at logical coordinates `(row, col)`, the local compute bank is selected by `row % CE` and `col % CE`, +while the spatial beat is calculated as `(row / CE) * NrComputeGroupsPerTileEdge + (col / CE)`. The final address is +`physical_slot * SpatialBeats + spatial_beat`. Therefore, each lane-local bank stores all spatial beats for all four +physical slots, and the same addressing rule is used by the FMA, VRF, and VLSU paths. + +The VRF and VLSU sides each have independent grouped read and write addresses, data and byte enables. These paths are +intended to access non-conflicting accumulator locations concurrently; for example, an FPU result may write one tile while +TV updates another tile and VTSE reads a third. The arbitration and hazard logic must still prevent overlapping accesses to +the same physical slot, spatial beat, and byte range. + +The grouped width is required by the VME tile-subset model; it is not an implementation-only widening of the accumulator +interface. A tile-subset specifier identifies an architectural tile and selects either a row or a column. Therefore, one +tile move instruction describes a logical row or column of the selected tile rather than one isolated accumulator word. The +OPE must consequently assemble or consume the complete active subset while preserving the mapping between the architectural +tile and the distributed compute array. A row is distributed over the eight compute lanes. If the tile edge is larger than +the compute-array edge, the row contains several compute groups along that edge, and each group contributes one consecutive +accumulator word in every lane-local bank. These words are therefore accessed as one aligned group. + +For an eight-by-eight compute array, a tile with edge 16 contains two compute groups along each edge. A row operation thus +groups two consecutive accumulator words per lane, giving eight lanes multiplied by two words and 32 bits per word, or +512 logical bits. With tile edge 32, the same architectural row contains four compute groups and the grouped width becomes +1024 bits. The corresponding column operation has the same architectural meaning; internally, the VLSU gathers or scatters +the required lane-local words and uses read-modify-write when only one column is being updated. These internal accesses are +implementation details of one architectural row or column operation, consistent with the TSS semantics. + +This also explains why grouped width must not be confused with a single-cycle VRF transfer. The grouped width describes the +logical accumulator payload selected by the VME instruction. The shared VAU/VRF interface is only 256 bits wide, so a 512-bit +tile row is transferred over two VRF cycles and a 1024-bit tile row over four cycles. The VLSU-to-OPE tile interface can carry +the wider logical row, while the VRF path serializes that same row according to its physical port width. `RD_PORTS` and `WR_PORTS` do **not** count independent clients. They specify how many consecutive physical accumulator words -one grouped VRF/VLSU access transfers from a common aligned base address. With TE=16 and CE=8, `GroupsPerEdge=2`; across the -eight compute columns, two 32-bit words per lane form one 512-bit tile row: +one grouped VRF/VLSU access transfers from a common aligned base address. The implementation sets both values to +`NrComputeGroupsPerTileEdge`, which is `TE / CE`. With TE=16 and CE=8, two consecutive accumulator rows are grouped per +lane. Across the eight compute lanes, this forms one logical 512-bit tile row: + +```text +CE * NrComputeGroupsPerTileEdge * AccElemWidth = 8 * 2 * 32 = 512 bits +``` + +With TE=32 and CE=8, `NrComputeGroupsPerTileEdge=4`, so four consecutive accumulator rows are grouped per lane and the +logical tile-row width becomes 1024 bits: ```text -CE * GroupsPerEdge * AccElemWidth = 8 * 2 * 32 = 512 bits +CE * NrComputeGroupsPerTileEdge * AccElemWidth = 8 * 4 * 32 = 1024 bits ``` +The VRF word itself is only 256 bits because the vector register file is shared with the VAU through three read ports and +one write port. Consequently, a TE=16 tile row requires two 256-bit VAU/VRF transfers, while a TE=32 tile row requires four. +The VLSU-to-OPE tile-row data width is independent of the VRF word width and is directly determined by `TE * AccElemWidth`: +512 bits for TE=16 and 1024 bits for TE=32. + The implementation derives `AddrWidth`, `RdPortIdxWidth` and `WrPortIdxWidth` from `DEPTH`, `RD_PORTS` and `WR_PORTS`. -Grouped base addresses are aligned to the corresponding number of consecutive words. +Grouped read and write addresses are aligned to the corresponding number of consecutive words, so the grouped transfer starts +at a common aligned base address and accesses `base + 0` through `base + NrComputeGroupsPerTileEdge - 1`. Independent ports do not remove data dependencies. Two writes must serialize when they target an overlapping physical tile/beat/byte footprint. Different logical tile IDs can still share a physical 32-bit word under TEW8 or TEW16, so conflict @@ -270,6 +350,28 @@ Move elements-per-word counters use `vrf_elem_count_t` so VRFWordWidth/8 fits ev ## 5. External Tile Path: VLSU +### Vector Memory EEW + +The decoder records the element width encoded by standard vector load/store +instructions in `op_mem.ew`. Unit-stride and strided accesses interpret this +field as data EEW. Indexed accesses retain the existing split: `op_mem.ew` is +the index EEW and VSEW is the data EEW. + +Both VLSU implementations derive a common data EEW with that rule. The data +EEW controls conversion of VL/VSTART to byte counts, element size, stride +normalization, load commit metadata, store strobes and memory request size. +Consequently, VLE32 transfers 32-bit elements even while the current matrix +VTYPE has VSEW=16. This removes the software-only VTYPE switch that was +previously needed around each packed load. + +The controller continues to install the current architectural VTYPE when an +LSU request is dispatched; the decoder's memory EEW survives separately in +`op_mem.ew`. For VTFMM, the scoreboard still records the architectural +implicit row-stride vector-register reads according to VSEW and TK, even +though widening SDOTP obtains both packed FP16 lanes from each base vector +element. `msetmtype` updates matrix VTYPE/mtype and clears VL; `msettn` restores +TN and VL with the LMUL/ETE limits applied. + VTLE/VTSE run in the VLSU, not in the OPE's four request queues. VLSU owns TSS interpretation, memory addressing, byte strobes, transactions and completion. OPE owns accumulator access, byte selection and resident/busy protection. The VLSU-facing read/write path is independent of @@ -344,8 +446,9 @@ For example, FP32 with LMUL=2 and `tk=2` records `{v0,v1,v8,v9}` for a source ba entered in `read_table` for WAR protection. LMUL expands the VRF footprint protected by the scoreboard; it does not independently add OPE MAC steps. MAC execution is -still controlled by `tm`, `tn` and `tk`: spatial beat address generation crosses vector-register boundaries naturally, while -`tk` selects the separately strided K rows. +controlled by `tm` and `tn`: spatial beat address generation crosses vector-register boundaries naturally. +For same-width FMA, `tk` selects separately strided K rows; widening SDOTP consumes +the packed K lanes in each base-vector element in parallel, without extra reduction beats. ### Address-Aware VRF Chaining in `DOUBLE_BW` @@ -398,7 +501,41 @@ VTDISCARD's all-ones mask is independent of TEW and TSS. It waits for older tile and younger tile operations wait for discard retirement. Discard is not a MAC and does not use the MAC/MAC exemption. OPE drain/idle guards remain necessary because an older MAC can respond before all its pipeline data is retired. -## 8. Maintenance and Verification +## 8. Packed FP16 Software + +The TE16 rolled FP16xFP16-to-FP32 generators first quantize A and B to FP16, +then compute the golden C with FP32 operands and accumulation. They pack two +adjacent reduction rows into each `uint32_t` word: + +```text +word[31:16] = K[2p] +word[15:0] = K[2p+1] +``` + +Both operands use this order, so one widening SDOTP performs the two matching +products. The generated layouts are `A[m_tile][K/2][TE]` and +`B[n_tile][K/2][TE]`; no FP16 value is converted to FP32 in memory. + +`matmul_l.K`, `MATMUL_K_CHUNK`, `valid_k`, and the K argument passed to +`matmul_fp16_fp32` all remain original FP16 reduction counts. Only packed-memory +calculations divide K by two: DMA extents and source offsets count packed +32-bit words. The kernel A/B parameters are `const __fp16 *`, so a complete +tile advances by `TE*K` halfwords. Each four-instruction VTFMM phase consumes +eight original K rows, hence the middle-loop count is `K/8 - 3`. Every VLE32 +transfers packed words directly; C and the golden reference remain FP32. + +The rolled schedules follow their FP32 counterparts after removing the +implicit row-stride VLE/VTFMM operations. Store-afterK distributes individual +VLE32 operations between VTFMM instructions and stores four completed tiles +after the K loop. Store-interleave overlaps tile stores, vector loads and +VTFMMs and retains its cross-block path. The vsetvl schedule replaces four +individual loads with one e32,m4 grouped load, restoring matrix state before +VTFMM as described above. + +Utilization counts real FP16 FMAs. Peak throughput is `CE * CE * 2` FMAs per +cycle because each CE lane consumes two packed FP16 products per SDOTP. + +## 9. Maintenance and Verification Shared configuration comes from `hw/ip/spatz/src/spatz_pkg.sv.tpl`; do not edit only the generated package. The cluster template currently links to this source. Check CE/TE, VRF operand width, accumulator depth and tile bandwidth together. diff --git a/hw/ip/spatz/src/ope/spatz_ope.sv b/hw/ip/spatz/src/ope/spatz_ope.sv index c8dab79ee..24ee4484f 100644 --- a/hw/ip/spatz/src/ope/spatz_ope.sv +++ b/hw/ip/spatz/src/ope/spatz_ope.sv @@ -46,16 +46,15 @@ module spatz_ope output logic tile_wready_o , input logic tile_rvalid_i , input tile_r_req_t tile_r_req_i , - output tile_row_t tile_rdata_o , + output tile_word_t tile_rdata_o , output logic tile_rready_o ); `include "common_cells/registers.svh" localparam NumPipeRegs = 4; - localparam GroupsPerEdge = TE / CE; // TE & CE should be power of 2 - localparam SpatialBeats = GroupsPerEdge * GroupsPerEdge; - localparam GroupIdxW = (GroupsPerEdge > 1) ? $clog2(GroupsPerEdge) : 1; + localparam SpatialBeats = NrAccPerComputeElem * NrAccPerComputeElem; + localparam GroupIdxW = (NrAccPerComputeElem > 1) ? $clog2(NrAccPerComputeElem) : 1; localparam SpatialBeatW = (SpatialBeats > 1) ? $clog2(SpatialBeats) : 1; localparam ReductionIdxW = (KMAX > 1) ? $clog2(KMAX) : 1; localparam NrAccumulatorTiles = NrPhysicalTile / AccElemBytes; @@ -74,13 +73,18 @@ module spatz_ope /* Shared Types and Signals */ /*------------------------------------------------------------*/ - typedef logic [$clog2(GroupsPerEdge+1)-1:0] group_count_t; + typedef logic [$clog2(NrAccPerComputeElem+1)-1:0] group_count_t; typedef logic [$clog2(MaxVtGroupWords+1)-1:0] vt_word_count_t; typedef logic [$clog2(VtMaxElemsPerWord+1)-1:0] vrf_elem_count_t; typedef logic [3:0] vreg_group_count_t; typedef logic [$clog2((TE<<1)+1)-1:0] tile_sum_t; typedef logic [5:0] operand_width_t; typedef logic [$clog2(VRFWordWidth)-1:0] vrf_bit_offset_t; + typedef struct packed { + logic [AccTileIdxW-1:0] tile; + logic [AccByteIdxW-1:0] byte_offset; + logic [AccElemBytes-1:0] byte_enable; + } acc_sel_t; typedef struct packed { spatz_id_t id; vreg_t vs1; @@ -110,6 +114,7 @@ module spatz_ope spatz_req_t spatz_req_mac ; logic mac_req_valid ; logic mac_req_ready ; + logic mac_req_eligible; spatz_req_t spatz_req_tv ; logic tv_req_valid ; logic tv_req_ready ; @@ -174,6 +179,7 @@ module spatz_ope logic vt_done_valid , vt_done_ready ; logic [$clog2(MaxVtGroupWords)-1:0] vt_word_idx_d, vt_word_idx_q; logic vt_word_fire; + logic vt_finishing; vt_word_count_t vt_active_words; vlen_t vt_active_elems; tile_count_t vt_active_lines; @@ -188,6 +194,7 @@ module spatz_ope ope_rsp_t tv_commit_rsp, tv_done_rsp ; logic tv_done_valid , tv_done_ready ; logic [$clog2(MaxVtGroupWords)-1:0] tv_word_idx_d, tv_word_idx_q; + logic tv_finishing; vt_word_count_t tv_active_words; vlen_t tv_active_elems; tile_count_t tv_active_lines; @@ -219,14 +226,21 @@ module spatz_ope // Accumulator datapath and access arbitration. logic [CE-1:0][CE-1:0][AccElemWidth-1:0] fma_addend; + acc_sel_t mac_acc_sel; + acc_sel_t fma_acc_sel; + acc_sel_t tv_acc_sel; + acc_sel_t vt_acc_sel; + acc_sel_t tile_r_acc_sel; + acc_sel_t tile_w_acc_sel; + acc_sel_t clean_acc_sel; logic [CE-1:0][CE-1:0][AccElemWidth-1:0] fma_result; logic [CE-1:0][CE-1:0] fma_result_valid; logic mac_use_sdotp; logic [CE-1:0][CE-1:0][AccElemWidth-1:0] acc_fma_rdata, acc_fma_wdata; logic [CE-1:0][CE-1:0][AccElemBytes-1:0] acc_fma_wen; - logic [CE-1:0][CE-1:0][GroupsPerEdge-1:0][AccElemWidth-1:0] acc_vrf_rdata, acc_vrf_wdata; + logic [CE-1:0][CE-1:0][NrAccPerComputeElem-1:0][AccElemWidth-1:0] acc_vrf_rdata, acc_vrf_wdata; logic [CE-1:0][CE-1:0][AccElemBytes-1:0] acc_vrf_wen; - logic [CE-1:0][CE-1:0][GroupsPerEdge-1:0][AccElemWidth-1:0] acc_vlsu_rdata, acc_vlsu_wdata; + logic [CE-1:0][CE-1:0][NrAccPerComputeElem-1:0][AccElemWidth-1:0] acc_vlsu_rdata, acc_vlsu_wdata; logic [CE-1:0][CE-1:0][AccElemBytes-1:0] acc_vlsu_wen; logic [AccAddrW-1:0] acc_fma_raddr, acc_fma_waddr; logic [AccAddrW-1:0] acc_vrf_raddr, acc_vrf_waddr; @@ -302,24 +316,27 @@ module spatz_ope // Shared Request Arbitration always_comb begin : req_ready_proc - mac_req_ready = 1'b0; + mac_req_eligible = 1'b0; vt_req_ready = 1'b0; tv_req_ready = 1'b0; clean_req_ready = 1'b0; // Simple commands such as VTZERO/VTDISCARD must complete before a younger MAC may start. if (clean_req_valid) clean_req_ready = clean_commit_ready && clean_acc_ready; - else if (vt_req_valid && !vt_busy_q && !resident_drain_valid_q && !mac_tile_busy[spatz_req_vt.op_ope.tss.tile_id] && + else if (vt_req_valid && (!vt_busy_q || (vt_finishing && vt_commit_ready)) && + !resident_drain_valid_q && !mac_tile_busy[spatz_req_vt.op_ope.tss.tile_id] && (!resident_valid_q || (spatz_req_vt.op_ope.tss.tile_id != resident_tile_q))) vt_req_ready = 1'b1; - else if (tv_req_valid && !tv_busy_q && !resident_drain_valid_q && !mac_tile_busy[spatz_req_tv.op_ope.tss.tile_id] && + else if (tv_req_valid && (!tv_busy_q || (tv_finishing && tv_commit_ready)) && + !resident_drain_valid_q && !mac_tile_busy[spatz_req_tv.op_ope.tss.tile_id] && (!resident_valid_q || (spatz_req_tv.op_ope.tss.tile_id != resident_tile_q))) tv_req_ready = 1'b1; else if (mac_req_valid) - mac_req_ready = mac_op_queue_ready && !resident_drain_valid_q && !(tile_rvalid_i && - (tile_r_req_i.idx == spatz_req_mac.mtd)); + mac_req_eligible = !resident_drain_valid_q && !(tile_rvalid_i && (tile_r_req_i.idx == spatz_req_mac.mtd)); end : req_ready_proc + assign mac_req_ready = mac_req_eligible && mac_op_queue_ready; + assign spatz_req_ready_o = (~mac_in_ready || ~tv_in_ready || ~vt_in_ready || ~clean_in_ready) ? 1'b0 : 1'b1; //////////////////////////////////////////////////////////////////////// @@ -331,8 +348,8 @@ module spatz_ope // active instruction and one look-ahead instruction instead of indexing a // context slot by every architectural tile. - assign mac_group_row = GroupIdxW'(mac_op_beat / GroupsPerEdge); - assign mac_group_col = GroupIdxW'(mac_op_beat % GroupsPerEdge); + assign mac_group_row = GroupIdxW'(mac_op_beat / NrAccPerComputeElem); + assign mac_group_col = GroupIdxW'(mac_op_beat % NrAccPerComputeElem); always_comb begin : mac_active_shape mac_active_tm = (mac_op_exec.tm < TE) ? mac_op_exec.tm : tile_count_t'(TE); @@ -344,7 +361,7 @@ module spatz_ope if (({1'b0, mac_group_col} + 1'b1) < mac_n_groups) mac_next_spatial_beat = mac_op_beat + 1'b1; else - mac_next_spatial_beat = SpatialBeatW'(({1'b0, mac_group_row} + 1'b1) * GroupsPerEdge); + mac_next_spatial_beat = SpatialBeatW'(({1'b0, mac_group_row} + 1'b1) * NrAccPerComputeElem); mac_lane_active = $bits(mac_lane_active)'(0); for (int unsigned row = 0; row < CE; row++) begin @@ -356,7 +373,7 @@ module spatz_ope end : mac_active_shape always_comb begin : mac_op_scheduler - logic result_match, result_switch; + logic result_match, result_switch, result_can_drain; logic tile_hazard; logic current_ready; @@ -370,10 +387,15 @@ module spatz_ope (mac_op_current_q.op.tile != resident_tile_q); end tile_hazard = fma_pipe_tile_busy[mac_op_current_q.op.tile][mac_beat_q] && !result_match; + result_can_drain = fma_pipe_result_valid && fma_pipe_drain; current_ready = mac_op_current_q.valid && (mac_current_empty || (!resident_drain_valid_q && !tile_hazard && !(tile_rvalid_i && (tile_r_req_i.idx == mac_op_current_q.op.tile)) && - ((!(&fma_pipe_valid)) || result_match || result_switch || fma_pipe_result_fire))); - mac_op_bypass = !mac_op_current_q.valid && mac_req_valid && mac_op_queue_ready && mac_req_ready && !mac_req_empty; + ((!(&fma_pipe_valid)) || result_match || result_switch || result_can_drain))); + // A dependent MAC must wait in the registered scheduler until the prior + // result reaches the forwarding stage. + mac_op_bypass = !mac_op_current_q.valid && !mac_op_next_q.valid && + mac_req_valid && mac_req_eligible && !mac_req_empty && + !fma_pipe_tile_busy[spatz_req_mac.mtd][SpatialBeatW'(0)]; mac_op_exec_valid = current_ready || mac_op_bypass; end : mac_op_scheduler @@ -708,27 +730,24 @@ module spatz_ope end assign vt_word_fire = vt_busy_q && vt_req_q.use_vd && vrf_wvalid_i; + assign vt_finishing = vt_busy_q && ((vt_active_words == 0) || + (vt_word_fire && (({1'b0, vt_word_idx_q} + 1'b1) >= vt_active_words))); always_comb begin : vt_handler vt_busy_d = vt_busy_q; vt_word_idx_d = vt_word_idx_q; - vt_commit_valid = 1'b0; - if (vt_req_valid && vt_req_ready) begin - vt_busy_d = 1'b1; - vt_word_idx_d = $bits(vt_word_idx_d)'(0); - end + vt_commit_valid = vt_finishing; - if (vt_busy_q && (vt_active_words == 0)) begin - vt_commit_valid = 1'b1; - end else if (vt_word_fire) begin - if (({1'b0, vt_word_idx_q} + 1'b1) >= vt_active_words) - vt_commit_valid = 1'b1; - else - vt_word_idx_d = vt_word_idx_q + 1'b1; - end + if (vt_word_fire && !vt_finishing) + vt_word_idx_d = vt_word_idx_q + 1'b1; - if (vt_commit_valid && vt_commit_ready) begin - vt_busy_d = 1'b0; + if (vt_finishing && vt_commit_ready) begin + // Retire the current move and accept the next one in the same cycle. + // This keeps vrf_we_o asserted across back-to-back VTMV_V_T moves. + vt_busy_d = vt_req_valid && vt_req_ready; + vt_word_idx_d = $bits(vt_word_idx_d)'(0); + end else if (!vt_busy_q && vt_req_valid && vt_req_ready) begin + vt_busy_d = 1'b1; vt_word_idx_d = $bits(vt_word_idx_d)'(0); end end : vt_handler @@ -800,27 +819,24 @@ module spatz_ope assign tv_vrf_data = tv_data_latched_q ? tv_data_q : vrf_rdata_i[2]; assign tv_acc_wen = tv_busy_q && tv_vrf_avail && !vt_busy_q; + assign tv_finishing = tv_busy_q && ((tv_active_words == 0) || + (tv_acc_wen && (({1'b0, tv_word_idx_q} + 1'b1) >= tv_active_words))); always_comb begin : tv_handler tv_busy_d = tv_busy_q; tv_word_idx_d = tv_word_idx_q; - tv_commit_valid = 1'b0; - if (tv_req_valid && tv_req_ready) begin - tv_busy_d = 1'b1; - tv_word_idx_d = $bits(tv_word_idx_d)'(0); - end + tv_commit_valid = tv_finishing; - if (tv_busy_q && (tv_active_words == vt_word_count_t'(0))) - tv_commit_valid = 1'b1; - else if (tv_acc_wen) begin - if (({1'b0, tv_word_idx_q} + 1'b1) >= tv_active_words) - tv_commit_valid = 1'b1; - else - tv_word_idx_d = tv_word_idx_q + 1'b1; - end + if (tv_acc_wen && !tv_finishing) + tv_word_idx_d = tv_word_idx_q + 1'b1; - if (tv_commit_valid && tv_commit_ready) begin - tv_busy_d = 1'b0; + if (tv_finishing && tv_commit_ready) begin + // Retire the current move and accept the next one in the same cycle. + // This keeps the VRF read path active across back-to-back VTMV_T_V moves. + tv_busy_d = tv_req_valid && tv_req_ready; + tv_word_idx_d = $bits(tv_word_idx_d)'(0); + end else if (!tv_busy_q && tv_req_valid && tv_req_ready) begin + tv_busy_d = 1'b1; tv_word_idx_d = $bits(tv_word_idx_d)'(0); end end : tv_handler @@ -922,6 +938,8 @@ module spatz_ope /*------------------------------------------------------------*/ always_comb begin : vrf_wr_proc + logic [VtMaxElemsPerWord-1:0][AccElemWidth-1:0] lane_data; + logic [VtMaxElemsPerWord-1:0] lane_valid; tile_dim_t idx; tile_dim_t row_i; tile_dim_t col_i; @@ -936,6 +954,8 @@ module spatz_ope vrf_we_o = 1'b0; vrf_wbe_o = vrf_be_t'(0); vrf_wdata_o = vrf_data_t'(0); + lane_data = '0; + lane_valid = '0; row_i = tile_dim_t'(0); col_i = tile_dim_t'(0); @@ -974,47 +994,47 @@ module spatz_ope acc_word = acc_vrf_rdata[row_i % CE][col_i % CE][col_i / CE]; for (int unsigned byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin - if (acc_zero_q[vt_acc_sel.tile][(row_i / CE) * GroupsPerEdge + (col_i / CE)][byte_idx]) + if (acc_zero_q[vt_acc_sel.tile][(row_i / CE) * NrAccPerComputeElem + (col_i / CE)][byte_idx]) acc_word[byte_idx*8 +: 8] = 8'd0; end - unique case (vt_req_q.op_ope.tew) - EW_8: begin - vrf_wdata_o[lane*8 +: 8] = acc_word[vt_acc_sel.byte_offset*8 +: 8]; - vrf_wbe_o[lane] = 1'b1; - end - EW_16: begin - vrf_wdata_o[lane*16 +: 16] = acc_word[vt_acc_sel.byte_offset*8 +: 16]; - vrf_wbe_o[lane*2 +: 2] = 2'b11; - end - default: begin - vrf_wdata_o[lane*AccElemWidth +: AccElemWidth] = acc_word; - vrf_wbe_o[lane*AccElemBytes +: AccElemBytes] = AccFullMask; - end - endcase + lane_data[lane] = acc_word; + lane_valid[lane] = 1'b1; end end end + unique case (vt_req_q.op_ope.tew) + EW_8: begin + for (int lane = 0; lane < VRFWordWidth / 8; lane++) begin + if (lane_valid[lane]) begin + vrf_wdata_o[lane*8 +: 8] = lane_data[lane][vt_acc_sel.byte_offset*8 +: 8]; + vrf_wbe_o[lane] = 1'b1; + end + end + end + EW_16: begin + for (int lane = 0; lane < VRFWordWidth / 16; lane++) begin + if (lane_valid[lane]) begin + vrf_wdata_o[lane*16 +: 16] = lane_data[lane][vt_acc_sel.byte_offset*8 +: 16]; + vrf_wbe_o[lane*2 +: 2] = 2'b11; + end + end + end + default: begin + for (int lane = 0; lane < VRFWordWidth / AccElemWidth; lane++) begin + if (lane_valid[lane]) begin + vrf_wdata_o[lane*AccElemWidth +: AccElemWidth] = lane_data[lane]; + vrf_wbe_o[lane*AccElemBytes +: AccElemBytes] = AccFullMask; + end + end + end + endcase end /*------------------------------------------------------------*/ /* Shared Accumulator Access and Tile Slice Selection */ /*------------------------------------------------------------*/ - typedef struct packed { - logic [AccTileIdxW-1:0] tile; - logic [AccByteIdxW-1:0] byte_offset; - logic [AccElemBytes-1:0] byte_enable; - } acc_sel_t; - // Accumulator slice selection. - acc_sel_t mac_acc_sel; - acc_sel_t fma_acc_sel; - acc_sel_t tv_acc_sel; - acc_sel_t vt_acc_sel; - acc_sel_t tile_r_acc_sel; - acc_sel_t tile_w_acc_sel; - acc_sel_t clean_acc_sel; - // Decode each architectural tile exactly once. The physical accumulator is // 32 bits wide; TEW8 selects a byte, TEW16 a half-word, and TEW32 the full word. always_comb begin : acc_select_proc @@ -1132,7 +1152,7 @@ module spatz_ope end bank_row = vt_req_q.op_ope.tss.is_row ? GroupIdxW'((vt_req_q.op_ope.tss.index + line_idx) / CE) : GroupIdxW'((line_word_idx * vt_elems_per_word) / CE); - acc_vrf_raddr = AccAddrW'(vt_acc_sel.tile * SpatialBeats + bank_row * GroupsPerEdge); + acc_vrf_raddr = AccAddrW'(vt_acc_sel.tile * SpatialBeats + bank_row * NrAccPerComputeElem); end else if (tv_busy_q) begin if (tv_line_words != 0) begin line_idx = tile_dim_t'(tv_word_idx_q / tv_line_words); @@ -1140,15 +1160,15 @@ module spatz_ope end bank_row = tv_req_q.op_ope.tss.is_row ? GroupIdxW'((tv_req_q.op_ope.tss.index + line_idx) / CE) : GroupIdxW'((line_word_idx * tv_elems_per_word) / CE); - acc_vrf_raddr = AccAddrW'(tv_acc_sel.tile * SpatialBeats + bank_row * GroupsPerEdge); + acc_vrf_raddr = AccAddrW'(tv_acc_sel.tile * SpatialBeats + bank_row * NrAccPerComputeElem); end if (tile_rvalid_i && tile_rready_o) begin bank_row = GroupIdxW'(tile_r_req_i.row / CE); - acc_vlsu_raddr = AccAddrW'(tile_r_acc_sel.tile * SpatialBeats + bank_row * GroupsPerEdge); + acc_vlsu_raddr = AccAddrW'(tile_r_acc_sel.tile * SpatialBeats + bank_row * NrAccPerComputeElem); end else if (tile_wvalid_i && tile_wready_o) begin bank_row = GroupIdxW'(tile_w_req_i.row / CE); - acc_vlsu_raddr = AccAddrW'(tile_w_acc_sel.tile * SpatialBeats + bank_row * GroupsPerEdge); + acc_vlsu_raddr = AccAddrW'(tile_w_acc_sel.tile * SpatialBeats + bank_row * NrAccPerComputeElem); end end : acc_read_addr_proc @@ -1162,6 +1182,8 @@ module spatz_ope vlen_t flat_elem_idx; tile_dim_t line_idx; tile_dim_t line_word_idx; + logic [2:0] tile_w_elem_bytes; + logic [2:0] tile_r_elem_bytes; acc_fma_wdata = $bits(acc_fma_wdata)'(0); acc_fma_wen = $bits(acc_fma_wen)'(0); @@ -1173,7 +1195,7 @@ module spatz_ope acc_vlsu_wen = $bits(acc_vlsu_wen)'(0); acc_vlsu_waddr = AccAddrW'(0); acc_flush = 1'b0; - tile_rdata_o = tile_row_t'(0); + tile_rdata_o = tile_word_t'(0); idx = tile_dim_t'(0); row_i = tile_dim_t'(0); @@ -1183,6 +1205,8 @@ module spatz_ope flat_elem_idx = vlen_t'(0); line_idx = tile_dim_t'(0); line_word_idx = tile_dim_t'(0); + tile_w_elem_bytes = 3'(1 << int'(tile_w_req_i.tew)); + tile_r_elem_bytes = 3'(1 << int'(tile_r_req_i.tew)); if (clean_req_valid && clean_req_ready && (spatz_req_clean.op_ope.is_discard)) begin acc_flush = 1'b1; @@ -1200,10 +1224,10 @@ module spatz_ope acc_vrf_waddr = acc_vrf_raddr; for (int row = 0; row < CE; row++) begin for (int col = 0; col < CE; col++) begin - for (int beat = 0; beat < GroupsPerEdge; beat++) begin + for (int beat = 0; beat < NrAccPerComputeElem; beat++) begin acc_vrf_wdata[row][col][beat] = acc_vrf_rdata[row][col][beat]; for (int byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin - if (acc_zero_q[tv_acc_sel.tile][bank_row * GroupsPerEdge + beat][byte_idx]) + if (acc_zero_q[tv_acc_sel.tile][bank_row * NrAccPerComputeElem + beat][byte_idx]) acc_vrf_wdata[row][col][beat][byte_idx*8 +: 8] = 8'd0; end end @@ -1228,9 +1252,9 @@ module spatz_ope acc_vrf_wdata[row_i % CE][col_i % CE][col_i / CE][tv_acc_sel.byte_offset*8 +: 8] = tv_vrf_data[lane*8 +: 8]; else if (tv_req_q.op_ope.tew == EW_16) acc_vrf_wdata[row_i % CE][col_i % CE][col_i / CE][tv_acc_sel.byte_offset*8 +: 16] = - tv_vrf_data[lane*16 +: 16]; + 16'(tv_vrf_data >> (lane * 16)); else - acc_vrf_wdata[row_i % CE][col_i % CE][col_i / CE] = tv_vrf_data[lane*AccElemWidth +: AccElemWidth]; + acc_vrf_wdata[row_i % CE][col_i % CE][col_i / CE] = AccElemWidth'(tv_vrf_data >> (lane * AccElemWidth)); end end end @@ -1240,29 +1264,27 @@ module spatz_ope acc_vlsu_waddr = acc_vlsu_raddr; for (int row = 0; row < CE; row++) begin for (int col = 0; col < CE; col++) begin - for (int beat = 0; beat < GroupsPerEdge; beat++) begin + for (int beat = 0; beat < NrAccPerComputeElem; beat++) begin acc_vlsu_wdata[row][col][beat] = acc_vlsu_rdata[row][col][beat]; for (int byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin - if (acc_zero_q[tile_w_acc_sel.tile][bank_row * GroupsPerEdge + beat][byte_idx]) + if (acc_zero_q[tile_w_acc_sel.tile][bank_row * NrAccPerComputeElem + beat][byte_idx]) acc_vlsu_wdata[row][col][beat][byte_idx*8 +: 8] = 8'd0; end end acc_vlsu_wen[row][col] = tile_w_acc_sel.byte_enable; end end - for (int i = 0; i < TE; i++) begin - col_i = tile_dim_t'(i); - if (i < tile_w_req_i.elems) begin + for (int unsigned byte_idx = 0; byte_idx < TileWordBWidth; byte_idx++) begin + tile_word_elem_idx_t elem_idx_in_word; + logic [2:0] elem_byte_idx; + + elem_idx_in_word = tile_word_elem_idx_t'(byte_idx / tile_w_elem_bytes); + elem_byte_idx = 3'(byte_idx % tile_w_elem_bytes); + if (elem_idx_in_word < tile_w_req_i.elems) begin + col_i = tile_dim_t'(tile_w_req_i.col + elem_idx_in_word); acc_vlsu_wen[tile_w_req_i.row % CE][col_i % CE] = tile_w_acc_sel.byte_enable; - if (tile_w_req_i.tew == EW_8) - acc_vlsu_wdata[tile_w_req_i.row % CE][col_i % CE][col_i / CE][tile_w_acc_sel.byte_offset*8 +: 8] = - tile_w_req_i.data[i*8 +: 8]; - else if (tile_w_req_i.tew == EW_16) - acc_vlsu_wdata[tile_w_req_i.row % CE][col_i % CE][col_i / CE][tile_w_acc_sel.byte_offset*8 +: 16] = - tile_w_req_i.data[i*16 +: 16]; - else - acc_vlsu_wdata[tile_w_req_i.row % CE][col_i % CE][col_i / CE] = - tile_w_req_i.data[i*AccElemWidth +: AccElemWidth]; + acc_vlsu_wdata[tile_w_req_i.row % CE][col_i % CE][col_i / CE] + [(tile_w_acc_sel.byte_offset+elem_byte_idx)*8 +: 8] = tile_w_req_i.data[byte_idx*8 +: 8]; end end end @@ -1285,25 +1307,27 @@ module spatz_ope if (tile_rvalid_i && tile_rready_o) begin bank_row = GroupIdxW'(tile_r_req_i.row / CE); - for (int i = 0; i < TE; i++) begin - logic [$clog2(CE)-1:0] bank_col; - logic [GroupIdxW-1:0] group_col; - - bank_col = $clog2(CE)'(i % CE); - group_col = GroupIdxW'(i / CE); - if (i < tile_r_req_i.elems) begin + for (int unsigned byte_idx = 0; byte_idx < TileWordBWidth; byte_idx++) begin + tile_word_elem_idx_t elem_idx_in_word; + logic [2:0] elem_byte_idx; + + elem_idx_in_word = tile_word_elem_idx_t'(byte_idx / tile_r_elem_bytes); + elem_byte_idx = 3'(byte_idx % tile_r_elem_bytes); + if (elem_idx_in_word < tile_r_req_i.elems) begin + logic [$clog2(CE)-1:0] bank_col; + logic [GroupIdxW-1:0] group_col; logic [AccElemWidth-1:0] acc_word; + col_i = tile_dim_t'(tile_r_req_i.col + elem_idx_in_word); + bank_col = $clog2(CE)'(col_i % CE); + group_col = GroupIdxW'(col_i / CE); acc_word = acc_vlsu_rdata[tile_r_req_i.row % CE][bank_col][group_col]; - for (int unsigned byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin - if (acc_zero_q[tile_r_acc_sel.tile][bank_row * GroupsPerEdge + group_col][byte_idx]) - acc_word[byte_idx*8 +: 8] = 8'd0; + for (int unsigned acc_byte_idx = 0; acc_byte_idx < AccElemBytes; acc_byte_idx++) begin + if (acc_zero_q[tile_r_acc_sel.tile][bank_row * NrAccPerComputeElem + group_col][acc_byte_idx]) + acc_word[acc_byte_idx*8 +: 8] = 8'd0; end - unique case (tile_r_req_i.tew) - EW_8: tile_rdata_o[i*8 +: 8] = acc_word[tile_r_acc_sel.byte_offset*8 +: 8]; - EW_16: tile_rdata_o[i*16 +: 16] = acc_word[tile_r_acc_sel.byte_offset*8 +: 16]; - default: tile_rdata_o[i*AccElemWidth +: AccElemWidth] = acc_word; - endcase + tile_rdata_o[byte_idx*8 +: 8] = + acc_word[(tile_r_acc_sel.byte_offset+elem_byte_idx)*8 +: 8]; end end end @@ -1336,13 +1360,13 @@ module spatz_ope end bank_row = tv_req_q.op_ope.tss.is_row ? GroupIdxW'((tv_req_q.op_ope.tss.index + line_idx) / CE) : GroupIdxW'((line_word_idx * tv_elems_per_word) / CE); - for (int beat = 0; beat < GroupsPerEdge; beat++) - acc_zero_d[tv_acc_sel.tile][bank_row * GroupsPerEdge + beat] &= ~tv_acc_sel.byte_enable; + for (int beat = 0; beat < NrAccPerComputeElem; beat++) + acc_zero_d[tv_acc_sel.tile][bank_row * NrAccPerComputeElem + beat] &= ~tv_acc_sel.byte_enable; end if (tile_wvalid_i && tile_wready_o) begin bank_row = GroupIdxW'(tile_w_req_i.row / CE); - for (int beat = 0; beat < GroupsPerEdge; beat++) - acc_zero_d[tile_w_acc_sel.tile][bank_row * GroupsPerEdge + beat] &= ~tile_w_acc_sel.byte_enable; + for (int beat = 0; beat < NrAccPerComputeElem; beat++) + acc_zero_d[tile_w_acc_sel.tile][bank_row * NrAccPerComputeElem + beat] &= ~tile_w_acc_sel.byte_enable; end end end : acc_zero_update @@ -1352,8 +1376,8 @@ module spatz_ope opope_accumulator #( .DATA_WIDTH(AccElemWidth ), .DEPTH (AccDepth ), - .RD_PORTS (GroupsPerEdge), - .WR_PORTS (GroupsPerEdge) + .RD_PORTS (NrAccPerComputeElem), + .WR_PORTS (NrAccPerComputeElem) ) i_accumulator ( .clk_i (clk_i ), .rst_ni (rst_ni ), diff --git a/hw/ip/spatz/src/rvv_pkg.sv b/hw/ip/spatz/src/rvv_pkg.sv index bd9d10ea0..3afdcc22f 100644 --- a/hw/ip/spatz/src/rvv_pkg.sv +++ b/hw/ip/spatz/src/rvv_pkg.sv @@ -30,7 +30,7 @@ package rvv_pkg; typedef struct packed { logic vill; // bit[XLEN-1]: illegal flag - logic altfmt; // self-def bit[8]: alternate format for VME + logic altfmt; // bit[8]: config by msetmtype rs2[8] or msetmtypei instr[28] logic vma; // bit[7]: mask-agnostic logic vta; // bit[6]: tail-agnostic vew_e vsew; // bits[5:3]: SEW encoding diff --git a/hw/ip/spatz/src/spatz.sv b/hw/ip/spatz/src/spatz.sv index ff4bc59f3..154b161fa 100644 --- a/hw/ip/spatz/src/spatz.sv +++ b/hw/ip/spatz/src/spatz.sv @@ -106,7 +106,7 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( tile_w_req_t tile_w_req; logic tile_rvalid, tile_rready; tile_r_req_t tile_r_req; - tile_row_t tile_rdata; + tile_word_t tile_rdata; logic vlsu_req_ready; logic vlsu_rsp_valid, vlsu_rsp_buf_valid; @@ -668,17 +668,10 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( `ifdef DOUBLE_BW // Buffering of VLSU1 when conflicting with VLSU0 logic vlsu_buf_en, vlsu_buf_push, vlsu_buf_pop, vrf_vlsu_wvalid, vlsu_buf_full, vlsu_buf_empty; - logic vlsu_direct_rsp_valid, vlsu_buf_drain; assign vlsu_buf_en = sb_we[VLSU_VD_WD1] && (!vrf_wvalid[VLSU_VD_WD1] || (vrf_wvalid[VLSU_VD_WD1] && !vlsu_buf_empty)); assign vlsu_buf_push = vlsu_buf_en && !vlsu_buf_full; - // A direct VLSU completion and a buffered interface-1 completion can occur - // in the same cycle, while the controller has only one response input. Keep - // the buffered entry queued until the direct completion has been delivered. - assign vlsu_direct_rsp_valid = - vlsu_rsp_valid && !((vlsu_rsp.intf_id == 1'b1) && vlsu_buf_push); - assign vlsu_buf_drain = !vlsu_buf_empty && !vlsu_direct_rsp_valid; - assign vlsu_buf_pop = vrf_wvalid[VLSU_VD_WD1] && vlsu_buf_drain; + assign vlsu_buf_pop = vrf_wvalid[VLSU_VD_WD1] && !vlsu_buf_empty; assign vrf_vlsu_wvalid = sb_we[VLSU_VD_WD1] && !vlsu_buf_full; fifo_v3 #( @@ -697,8 +690,7 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( vrf_waddr[VLSU_VD_WD1], vrf_wbe [VLSU_VD_WD1], sb_id [SB_VLSU_VD_WD1], - vlsu_rsp, - vlsu_rsp_valid} ), + vlsu_rsp_t'('0), 1'b0} ), .push_i (vlsu_buf_push ), .data_o (vlsu_buf_data ), .pop_i (vlsu_buf_pop ) @@ -739,30 +731,12 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( end `ifdef DOUBLE_BW - // VLSU1 buffering - // Check if interface 1 response is being buffered, if so do not send response now - if (vlsu_rsp_valid && (vlsu_rsp.intf_id == 1'b1) && vlsu_buf_push) - vlsu_rsp_buf_valid = 1'b0; - if (!vlsu_buf_empty) begin - // Do not expose a newer interface-1 write while the FIFO head is held. - sb_we_buf[VLSU_VD_WD1] = 1'b0; - if (vlsu_buf_drain) begin - sb_we_buf [VLSU_VD_WD1] = 1'b1; - vrf_wdata_buf[VLSU_VD_WD1] = vlsu_buf_data.wdata; - vrf_waddr_buf[VLSU_VD_WD1] = vlsu_buf_data.waddr; - vrf_wbe_buf [VLSU_VD_WD1] = vlsu_buf_data.wbe; - sb_buf_id [SB_VLSU_VD_WD1] = vlsu_buf_data.wid; - if (vlsu_buf_data.rsp_valid) begin - vlsu_rsp_buf = vlsu_buf_data.rsp; - vlsu_rsp_buf_valid = vrf_wvalid[VLSU_VD_WD1]; - end - end - end else begin - // If the buffer is being enabled in this cycle, don't send the response now - if (vlsu_buf_en) begin - vlsu_rsp_buf_valid = 1'b0; - end + sb_we_buf [VLSU_VD_WD1] = 1'b1; + vrf_wdata_buf[VLSU_VD_WD1] = vlsu_buf_data.wdata; + vrf_waddr_buf[VLSU_VD_WD1] = vlsu_buf_data.waddr; + vrf_wbe_buf [VLSU_VD_WD1] = vlsu_buf_data.wbe; + sb_buf_id [SB_VLSU_VD_WD1] = vlsu_buf_data.wid; end `endif `endif diff --git a/hw/ip/spatz/src/spatz_controller.sv b/hw/ip/spatz/src/spatz_controller.sv index 6b5611c8a..3d4197446 100644 --- a/hw/ip/spatz/src/spatz_controller.sv +++ b/hw/ip/spatz/src/spatz_controller.sv @@ -134,7 +134,9 @@ module spatz_controller always_comb begin : proc_vcsr - automatic logic [$clog2(MAXVL):0] vlmax = 0; + logic [$clog2(MAXVL):0] vlmax; + + vlmax = '0; vstart_d = vstart_q; vl_d = vl_q; @@ -231,6 +233,8 @@ module spatz_controller int unsigned eve; int unsigned lmul_req, lmul_k, lmul_widen; + // Both configuration forms supply altfmt through the decoded vtype. + // Keep it while deriving matrix LMUL and the agnostic policy bits. vtype_d = spatz_req.vtype; invalid_vtype = @@ -244,8 +248,8 @@ module spatz_controller twiden = 1 << (spatz_req.mtype.mtwiden - 1); tewb = (1 << int'(vtype_d.vsew)) * twiden; - // SEW8 is valid for byte tile storage/moves, not FP8 arithmetic or widening yet. - if (((vtype_d.vsew == EW_8) && (spatz_req.mtype.mtwiden != 2'b01)) || (tewb > AccElemBytes)) begin + // The configured tile element must fit in one accumulator element. + if (tewb > AccElemBytes) begin invalid_vtype = 1'b1; end else begin unique case (vtype_d.vsew) @@ -693,12 +697,17 @@ module spatz_controller `ifdef DOUBLE_BW // Store the decisions for (int unsigned port = 0; port < NrVregfilePorts; port++) begin - automatic int unsigned port_idx = port - SB_VFU_VD_WD; + int unsigned port_idx; + + port_idx = port - SB_VFU_VD_WD; if (sb_enable_o[port]) begin // VFU and VSLDU: intID derived from vl_cnt progress, vl_cnt updated every successful write if (port inside {SB_VFU_VD_WD, SB_VSLDU_VD_WD}) begin - automatic logic intID = (vl_cnt_q[sb_id_i[port]] < vl_max_d[sb_id_i[port]]) ? 0 : 1; - automatic int VRFWriteSize = narrow_q[sb_id_i[port]] ? VRFWordBWidth >> 1 : VRFWordBWidth; + logic intID; + int VRFWriteSize; + + intID = (vl_cnt_q[sb_id_i[port]] < vl_max_d[sb_id_i[port]]) ? 0 : 1; + VRFWriteSize = narrow_q[sb_id_i[port]] ? VRFWordBWidth >> 1 : VRFWordBWidth; // Update vl_cnt if actually written into the VRF if (sb_wrote_result_i[port_idx]) @@ -711,8 +720,10 @@ module spatz_controller end // VLSU: intID is fixed per interface (0 for WD0, 1 for WD1) if (port inside {SB_VLSU_VD_WD0, SB_VLSU_VD_WD1}) begin - automatic int unsigned intID = port - SB_VLSU_VD_WD0; - automatic vlen_t next_vlsu_vl; + int unsigned intID; + vlen_t next_vlsu_vl; + + intID = port - SB_VLSU_VD_WD0; wrote_result_narrowing_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] ^ narrow_wide_q[sb_id_i[port]]; wrote_result_d[intID][sb_id_i[port]] = sb_wrote_result_i[port_idx] && (!narrow_wide_q[sb_id_i[port]] || wrote_result_narrowing_q[sb_id_i[port]]); next_vlsu_vl = vlsu_vl_cnt_q[intID][sb_id_i[port]]; @@ -729,7 +740,9 @@ module spatz_controller // Store the decisions for all write-destination ports: VFU, VLSU, VSLDU for (int unsigned port = 0; port < NrVregfilePorts; port++) begin if (sb_enable_o[port] && (port inside {SB_VFU_VD_WD, SB_VLSU_VD_WD, SB_VSLDU_VD_WD})) begin - automatic int unsigned port_idx = port - SB_VFU_VD_WD; + int unsigned port_idx; + + port_idx = port - SB_VFU_VD_WD; wrote_result_narrowing_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] ^ narrow_wide_q[sb_id_i[port]]; wrote_result_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] && (!narrow_wide_q[sb_id_i[port]] || wrote_result_narrowing_q[sb_id_i[port]]); end @@ -926,8 +939,8 @@ module spatz_controller if (spatz_req.use_vs2) begin for (int unsigned row = 0; row < KMAX; row++) begin for (int unsigned group_idx = 0; group_idx < 8; group_idx++) begin - automatic vreg_group_idx_t group_vreg = - {1'b0, spatz_req.vs2} + vreg_group_idx_t'(row * sb_vs_row_stride + group_idx); + vreg_group_idx_t group_vreg; + group_vreg = {1'b0, spatz_req.vs2} + vreg_group_idx_t'(row * sb_vs_row_stride + group_idx); if ((row < sb_vs_rows) && (group_idx < sb_vs_group_count) && (group_vreg < NRVREG)) begin @@ -941,13 +954,12 @@ module spatz_controller if (spatz_req.use_vs1) begin for (int unsigned row = 0; row < KMAX; row++) begin for (int unsigned group_idx = 0; group_idx < 8; group_idx++) begin - automatic vreg_group_idx_t group_vreg = - {1'b0, spatz_req.vs1} + vreg_group_idx_t'(row * sb_vs_row_stride + group_idx); + vreg_group_idx_t group_vreg; + group_vreg = {1'b0, spatz_req.vs1} + vreg_group_idx_t'(row * sb_vs_row_stride + group_idx); if ((row < sb_vs_rows) && (group_idx < sb_vs_group_count) && (group_vreg < NRVREG)) begin - scoreboard_d[spatz_req.id].deps[write_table_d[group_vreg].id] |= - write_table_d[group_vreg].valid; + scoreboard_d[spatz_req.id].deps[write_table_d[group_vreg].id] |= write_table_d[group_vreg].valid; read_table_d[group_vreg] = {spatz_req.id, 1'b1}; end end @@ -958,17 +970,16 @@ module spatz_controller // Vector stores encode their data source in vd (architectural vs3). // Reserve every physical register in the current LMUL group. for (int unsigned group_idx = 0; group_idx < 8; group_idx++) begin - automatic vreg_group_idx_t group_vreg = - {1'b0, spatz_req.vd} + vreg_group_idx_t'(group_idx); + vreg_group_idx_t group_vreg; + + group_vreg = {1'b0, spatz_req.vd} + vreg_group_idx_t'(group_idx); if ((group_idx < vreg_group_count) && (group_vreg < NRVREG)) begin - scoreboard_d[spatz_req.id].deps[write_table_d[group_vreg].id] |= - write_table_d[group_vreg].valid; + scoreboard_d[spatz_req.id].deps[write_table_d[group_vreg].id] |= write_table_d[group_vreg].valid; read_table_d[group_vreg] = {spatz_req.id, 1'b1}; end end end else begin - scoreboard_d[spatz_req.id].deps[write_table_d[spatz_req.vd].id] |= - write_table_d[spatz_req.vd].valid; + scoreboard_d[spatz_req.id].deps[write_table_d[spatz_req.vd].id] |= write_table_d[spatz_req.vd].valid; read_table_d[spatz_req.vd] = {spatz_req.id, 1'b1}; end end @@ -989,8 +1000,8 @@ module spatz_controller // Vector loads write the complete LMUL register group, not only the // aligned architectural base register encoded by vd. for (int unsigned group_idx = 0; group_idx < 8; group_idx++) begin - automatic vreg_group_idx_t group_vreg = - {1'b0, spatz_req.vd} + vreg_group_idx_t'(group_idx); + vreg_group_idx_t group_vreg; + group_vreg = {1'b0, spatz_req.vd} + vreg_group_idx_t'(group_idx); if ((group_idx < vreg_group_count) && (group_vreg < NRVREG)) begin scoreboard_d[spatz_req.id].deps[write_table_d[group_vreg].id] |= write_table_d[group_vreg].valid; @@ -1022,9 +1033,9 @@ module spatz_controller `ifdef DOUBLE_BW narrow_d[spatz_req.id] = spatz_req.op_arith.is_narrowing; - // Track request vl for vector chaining, used only for DOUBLE_BW - // Default spatz uses 1-bit credit counter using wrote_result_q for chaining - vl_max_d[spatz_req.id] = (spatz_req.vl >> 1) << spatz_req.vtype.vsew; + // Track the byte extent of each half-vector. Vector memory operations use their encoded EEW, + // which may differ from SEW for both loads and stores; arithmetic operations use the configured SEW. + vl_max_d[spatz_req.id] = (spatz_req.vl >> 1) << ((spatz_req.ex_unit == LSU) ? spatz_req.op_mem.ew : spatz_req.vtype.vsew); vl_cnt_d[spatz_req.id] = '0; `endif end @@ -1408,7 +1419,7 @@ module spatz_controller end else if (!issue_tile_aligned) begin issue_rsp_o.accept = 1'b0; end else if (decoder_rsp.spatz_req.op_ope.is_mac && (vtype_q.vsew == EW_8)) begin - // No FP8 compute lane is instantiated; reject instead of waiting for a missing result. + // No 8-bit MAC lane is instantiated; reject instead of waiting for a missing result. issue_rsp_o.accept = 1'b0; end else if ((decoder_rsp.spatz_req.op_ope.is_vt || decoder_rsp.spatz_req.op_ope.is_tv) && diff --git a/hw/ip/spatz/src/spatz_decoder.sv b/hw/ip/spatz/src/spatz_decoder.sv index 7d3c54a74..58b573396 100644 --- a/hw/ip/spatz/src/spatz_decoder.sv +++ b/hw/ip/spatz/src/spatz_decoder.sv @@ -49,7 +49,9 @@ module spatz_decoder // We have a new instruction that need to be decoded if (decoder_req_valid_i) begin // Retrieve the opcode - automatic logic [6:0] opcode = decoder_req_i.instr[6:0]; + logic [6:0] opcode; + + opcode = decoder_req_i.instr[6:0]; unique casez (decoder_req_i.instr) // --- Tile load/store (memory ↔ tile accumulators via LSU) --- @@ -141,14 +143,23 @@ module spatz_decoder riscv_instr::VSOXEI16_V, riscv_instr::VSOXEI32_V, riscv_instr::VSOXEI64_V: begin - automatic vreg_t ls_vd = decoder_req_i.instr[11:7]; - automatic vreg_t ls_rs1 = decoder_req_i.instr[19:15]; - automatic vreg_t ls_s2 = decoder_req_i.instr[24:20]; - automatic logic [2:0] ls_width = decoder_req_i.instr[14:12]; - automatic logic ls_vm = decoder_req_i.instr[25]; - automatic logic [1:0] ls_mop = decoder_req_i.instr[27:26]; - automatic logic ls_mew = decoder_req_i.instr[28]; - automatic logic [2:0] ls_nf = decoder_req_i.instr[31:29]; + vreg_t ls_vd; + vreg_t ls_rs1; + vreg_t ls_s2; + logic [2:0] ls_width; + logic ls_vm; + logic [1:0] ls_mop; + logic ls_mew; + logic [2:0] ls_nf; + + ls_vd = decoder_req_i.instr[11:7]; + ls_rs1 = decoder_req_i.instr[19:15]; + ls_s2 = decoder_req_i.instr[24:20]; + ls_width = decoder_req_i.instr[14:12]; + ls_vm = decoder_req_i.instr[25]; + ls_mop = decoder_req_i.instr[27:26]; + ls_mew = decoder_req_i.instr[28]; + ls_nf = decoder_req_i.instr[31:29]; // Retrieve VSEW unique case ({ls_mew, ls_width}) @@ -438,11 +449,17 @@ module spatz_decoder riscv_instr::VSLIDEDOWN_VX, riscv_instr::VSLIDEDOWN_VI, riscv_instr::VSLIDE1DOWN_VX: begin - automatic opcodev_func3_e func3 = opcodev_func3_e'(decoder_req_i.instr[14:12]); - automatic vreg_t arith_s1 = decoder_req_i.instr[19:15]; - automatic vreg_t arith_s2 = decoder_req_i.instr[24:20]; - automatic vreg_t arith_d = decoder_req_i.instr[11:7]; - automatic logic arith_vm = decoder_req_i.instr[25]; + opcodev_func3_e func3; + vreg_t arith_s1; + vreg_t arith_s2; + vreg_t arith_d; + logic arith_vm; + + func3 = opcodev_func3_e'(decoder_req_i.instr[14:12]); + arith_s1 = decoder_req_i.instr[19:15]; + arith_s2 = decoder_req_i.instr[24:20]; + arith_d = decoder_req_i.instr[11:7]; + arith_vm = decoder_req_i.instr[25]; spatz_req.op_arith.vm = arith_vm; spatz_req.op_sld.vm = arith_vm; @@ -927,8 +944,11 @@ module spatz_decoder // Move to the scalar RF riscv_instr::VMV_X_S: begin - automatic vreg_t arith_s2 = decoder_req_i.instr[24:20]; - automatic vreg_t arith_d = decoder_req_i.instr[11:7]; + vreg_t arith_s2; + vreg_t arith_d; + + arith_s2 = decoder_req_i.instr[24:20]; + arith_d = decoder_req_i.instr[11:7]; spatz_req.op = VADD; spatz_req.ex_unit = VFU; @@ -1031,11 +1051,17 @@ module spatz_decoder riscv_instr::VFSLIDE1UP_VF, riscv_instr::VFSLIDE1DOWN_VF: begin if (spatz_pkg::FPU) begin - automatic opcodev_func3_e func3 = opcodev_func3_e'(decoder_req_i.instr[14:12]); - automatic vreg_t arith_s1 = decoder_req_i.instr[19:15]; - automatic vreg_t arith_s2 = decoder_req_i.instr[24:20]; - automatic vreg_t arith_d = decoder_req_i.instr[11:7]; - automatic logic arith_vm = decoder_req_i.instr[25]; + opcodev_func3_e func3; + vreg_t arith_s1; + vreg_t arith_s2; + vreg_t arith_d; + logic arith_vm; + + func3 = opcodev_func3_e'(decoder_req_i.instr[14:12]); + arith_s1 = decoder_req_i.instr[19:15]; + arith_s2 = decoder_req_i.instr[24:20]; + arith_d = decoder_req_i.instr[11:7]; + arith_vm = decoder_req_i.instr[25]; spatz_req.op_arith.vm = arith_vm; spatz_req.op_sld.vm = arith_vm; @@ -1385,11 +1411,17 @@ module spatz_decoder `ifdef VENTAGLIO riscv_instr::VFXMACC_VRF: begin - automatic vreg_t vd_field = decoder_req_i.instr[11:7]; - automatic logic [4:0] rs1_field = decoder_req_i.instr[19:15]; // FP scalar reg id - automatic vreg_t vs2_field = decoder_req_i.instr[24:20]; // weight vreg - automatic vreg_t vs1_field = decoder_req_i.instr[31:27]; // index vreg - automatic logic [1:0] funct2 = decoder_req_i.instr[26:25]; + vreg_t vd_field; + logic [4:0] rs1_field; // FP scalar reg id + vreg_t vs2_field; // weight vreg + vreg_t vs1_field; // index vreg + logic [1:0] funct2; + + vd_field = decoder_req_i.instr[11:7]; + rs1_field = decoder_req_i.instr[19:15]; + vs2_field = decoder_req_i.instr[24:20]; + vs1_field = decoder_req_i.instr[31:27]; + funct2 = decoder_req_i.instr[26:25]; // Only fp32 for now; reserve other funct2 codes if (funct2 != 2'b00) illegal_instr = 1'b1; @@ -1420,10 +1452,15 @@ module spatz_decoder end riscv_instr::VFXMUL_VRF: begin - automatic vreg_t vd_field = decoder_req_i.instr[11:7]; - automatic vreg_t vs2_field = decoder_req_i.instr[24:20]; // weight - automatic vreg_t vs1_field = decoder_req_i.instr[31:27]; // index - automatic logic [1:0] funct2 = decoder_req_i.instr[26:25]; + vreg_t vd_field; + vreg_t vs2_field; // weight + vreg_t vs1_field; // index + logic [1:0] funct2; + + vd_field = decoder_req_i.instr[11:7]; + vs2_field = decoder_req_i.instr[24:20]; + vs1_field = decoder_req_i.instr[31:27]; + funct2 = decoder_req_i.instr[26:25]; // funct2=01 reserved for vfxmul (this op); other values illegal for now if (funct2 != 2'b01) illegal_instr = 1'b1; @@ -1459,7 +1496,9 @@ module spatz_decoder // walks every bank cell and writes 0. Routed via ex_unit=SLD so it // bypasses the VFU and retires through ventaglio's vtl_rsp_o. riscv_instr::VVENTCLR: begin - automatic logic [1:0] funct2 = decoder_req_i.instr[26:25]; + logic [1:0] funct2; + + funct2 = decoder_req_i.instr[26:25]; if (funct2 != 2'b11) illegal_instr = 1'b1; @@ -1479,8 +1518,11 @@ module spatz_decoder // Move to the scalar FP RF riscv_instr::VFMV_F_S: begin if (spatz_pkg::FPU) begin - automatic vreg_t arith_s2 = decoder_req_i.instr[24:20]; - automatic vreg_t arith_d = decoder_req_i.instr[11:7]; + vreg_t arith_s2; + vreg_t arith_d; + + arith_s2 = decoder_req_i.instr[24:20]; + arith_d = decoder_req_i.instr[11:7]; spatz_req.op = VADD; spatz_req.ex_unit = VFU; @@ -1955,10 +1997,15 @@ module spatz_decoder riscv_instr::CSRRWI, riscv_instr::CSRRSI, riscv_instr::CSRRCI: begin - automatic logic [11:0] csr_addr = decoder_req_i.instr[31:20]; - automatic vreg_t csr_rd = decoder_req_i.instr[11:7]; - automatic vreg_t csr_rs1 = decoder_req_i.instr[19:15]; - automatic logic csr_is_imm = decoder_req_i.instr[14]; + logic [11:0] csr_addr; + vreg_t csr_rd; + vreg_t csr_rs1; + logic csr_is_imm; + + csr_addr = decoder_req_i.instr[31:20]; + csr_rd = decoder_req_i.instr[11:7]; + csr_rs1 = decoder_req_i.instr[19:15]; + csr_is_imm = decoder_req_i.instr[14]; spatz_req.op = VCSR; spatz_req.ex_unit = CON; @@ -2038,8 +2085,11 @@ module spatz_decoder riscv_instr::VSETVL, riscv_instr::VSETVLI, riscv_instr::VSETIVLI: begin - automatic vreg_t setvl_rs1 = decoder_req_i.instr[19:15]; - automatic vreg_t setvl_rd = decoder_req_i.instr[11:7]; + vreg_t setvl_rs1; + vreg_t setvl_rd; + + setvl_rs1 = decoder_req_i.instr[19:15]; + setvl_rd = decoder_req_i.instr[11:7]; spatz_req.rd = setvl_rd; spatz_req.use_rd = 1'b1; @@ -2070,23 +2120,27 @@ module spatz_decoder // VME (Vector Matrix Extension) Instructions // ------------------------------------------------------- - riscv_instr::MSETMTYPE, - riscv_instr::MSETMTYPEI: begin + riscv_instr::MSETMTYPE: begin spatz_req.op = VMCFG; spatz_req.ex_unit = CON; spatz_req.use_rd = 1'b0; spatz_req.op_ope.cfg_sel = 2'b00; - if (decoder_req_i.instr[25]) begin - // register form: msetmtype - spatz_req.mtype = mtype_t'(decoder_req_i.rs1); - spatz_req.vtype = vtype_t'(decoder_req_i.rs2); - end else begin - // immediate form: msetmtypei - spatz_req.mtype = '0; - spatz_req.mtype[4:0] = decoder_req_i.instr[19:15]; - spatz_req.vtype = '0; - spatz_req.vtype.vsew = vew_e'(decoder_req_i.instr[24:22]); - end + spatz_req.mtype = mtype_t'(decoder_req_i.rs1); + // Register form carries altfmt in vtype bit 8; higher bits are reserved. + spatz_req.vtype = vtype_t'({1'b0, decoder_req_i.rs2[8:0]}); + spatz_req.vtype.vill = |decoder_req_i.rs2[$bits(decoder_req_i.rs2)-1:9]; + end + + riscv_instr::MSETMTYPEI: begin + spatz_req.op = VMCFG; + spatz_req.ex_unit = CON; + spatz_req.use_rd = 1'b0; + spatz_req.op_ope.cfg_sel = 2'b00; + spatz_req.mtype = '0; + spatz_req.mtype.mtwiden = decoder_req_i.instr[16:15]; + spatz_req.vtype = '0; + spatz_req.vtype.vsew = vew_e'({1'b0, decoder_req_i.instr[24:23]}); + spatz_req.vtype.altfmt = decoder_req_i.instr[28]; end riscv_instr::MSETTN, diff --git a/hw/ip/spatz/src/spatz_doublebw_vlsu.sv b/hw/ip/spatz/src/spatz_doublebw_vlsu.sv index e08a7300f..ed2792e34 100644 --- a/hw/ip/spatz/src/spatz_doublebw_vlsu.sv +++ b/hw/ip/spatz/src/spatz_doublebw_vlsu.sv @@ -53,7 +53,7 @@ module spatz_doublebw_vlsu output logic tile_rvalid_o, output tile_r_req_t tile_r_req_o, - input tile_row_t tile_rdata_i, + input tile_word_t tile_rdata_i, input logic tile_rready_i, // Memory Request output spatz_mem_req_t [NrMemPorts-1:0] spatz_mem_req_o, @@ -165,7 +165,7 @@ module spatz_doublebw_vlsu /* Tile Memory Types and State */ /*------------------------------------------------------------*/ - localparam int unsigned TileRowBytes = TileEdge * AccElemBytes; + localparam int unsigned TileRowBytes = TileRowBWidth; localparam int unsigned TileMaxChunks = (TileRowBytes + 2 * MemDataWidthB - 2) / MemDataWidthB; typedef logic [$clog2(TileRowBytes+1)-1:0] tile_byte_cnt_t; typedef logic [$clog2(TileMaxChunks+1)-1:0] tile_chunk_cnt_t; @@ -179,10 +179,11 @@ module spatz_doublebw_vlsu tile_dim_t row_idx; tile_count_t elems; logic is_col; + tile_dim_t word_col; tile_byte_cnt_t bytes; tile_chunk_cnt_t chunk_sent; tile_row_t packed_data; - tile_row_t row_buf; + tile_word_t row_buf; } tile_ctx_t; typedef struct packed { @@ -248,6 +249,7 @@ module spatz_doublebw_vlsu // Map an architectural tile to its first physical accumulator. assign tile_tss_req.idx = mem_spatz_req.op_ope.tss.tile_id; assign tile_tss_req.row = mem_spatz_req.op_ope.tss.index; + assign tile_tss_req.col = tile_dim_t'(0); assign tile_tss_req.tew = mem_spatz_req.op_ope.tew; logic [31:0] tile_mem_base_aligned; @@ -365,20 +367,53 @@ module spatz_doublebw_vlsu /* Tile Accumulator Data Shaping */ /*------------------------------------------------------------*/ - // One tile-port handshake transfers the whole selected row. A column - // request reads full physical rows so its fixed column index is present. + // A fixed-width Tile word can contain a runtime-dependent number of TEW + // elements. Rows wider than one Tile word are transferred in consecutive + // word segments; a column accesses the segment containing that column. + tile_count_t tile_word_capacity; tile_count_t tile_req_elems; - assign tile_req_elems = tile_ctx_q.is_col ? tile_count_t'(TileEdge) : tile_ctx_q.elems; + tile_dim_t tile_req_col; + + always_comb begin : proc_tile_word_geometry + tile_count_t remaining_elems; + int unsigned word_capacity; + + word_capacity = TileWordBWidth / tile_elem_bytes; + tile_word_capacity = (word_capacity > TileEdge) ? tile_count_t'(TileEdge) : tile_count_t'(word_capacity); + + if (tile_ctx_q.is_col) begin + tile_req_col = tile_dim_t'((tile_ctx_q.access.row / tile_word_capacity) * tile_word_capacity); + remaining_elems = tile_count_t'(TileEdge - tile_req_col); + end else begin + tile_req_col = tile_ctx_q.word_col; + remaining_elems = (tile_ctx_q.elems > tile_count_t'(tile_ctx_q.word_col)) ? + tile_ctx_q.elems - tile_count_t'(tile_ctx_q.word_col) : tile_count_t'(0); + end + tile_req_elems = (remaining_elems < tile_word_capacity) ? remaining_elems : tile_word_capacity; + end - tile_row_t tile_col_write_row; - always_comb begin - tile_byte_cnt_t dst_byte; + tile_word_t tile_row_write_word; + tile_word_t tile_col_write_word; + always_comb begin : proc_tile_write_word + tile_count_t local_col; + + tile_row_write_word = tile_word_t'(0); + tile_col_write_word = tile_ctx_q.row_buf; + local_col = tile_count_t'(tile_ctx_q.access.row - tile_req_col); + + for (int unsigned elem = 0; elem < TileEdge; elem++) begin + for (int unsigned byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin + if ((elem < tile_req_elems) && (byte_idx < tile_elem_bytes)) begin + tile_row_write_word[(elem*tile_elem_bytes+byte_idx)*8 +: 8] = + tile_ctx_q.packed_data[((tile_req_col+elem)*tile_elem_bytes+byte_idx)*8 +: 8]; + end + end + end - tile_col_write_row = tile_ctx_q.row_buf; - dst_byte = tile_byte_cnt_t'(tile_ctx_q.access.row * tile_elem_bytes); for (int unsigned byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin if (byte_idx < tile_elem_bytes) begin - tile_col_write_row[(dst_byte+byte_idx)*8 +: 8] = tile_ctx_q.packed_data[(tile_ctx_q.row_idx*tile_elem_bytes+byte_idx)*8 +: 8]; + tile_col_write_word[(local_col*tile_elem_bytes+byte_idx)*8 +: 8] = + tile_ctx_q.packed_data[(tile_ctx_q.row_idx*tile_elem_bytes+byte_idx)*8 +: 8]; end end end @@ -408,22 +443,18 @@ module spatz_doublebw_vlsu tile_wvalid_o = 1'b0; tile_w_req_o.idx = tile_ctx_q.access.idx; tile_w_req_o.row = tile_ctx_q.is_col ? tile_ctx_q.row_idx : tile_ctx_q.access.row; + tile_w_req_o.col = tile_req_col; tile_w_req_o.elems = tile_req_elems; tile_w_req_o.tew = tile_ctx_q.access.tew; - tile_w_req_o.data = tile_ctx_q.is_col ? tile_col_write_row : tile_ctx_q.packed_data; + tile_w_req_o.data = tile_ctx_q.is_col ? tile_col_write_word : tile_row_write_word; tile_rvalid_o = 1'b0; tile_r_req_o.idx = tile_ctx_q.access.idx; tile_r_req_o.row = tile_ctx_q.is_col ? tile_ctx_q.row_idx : tile_ctx_q.access.row; + tile_r_req_o.col = tile_req_col; tile_r_req_o.elems = tile_req_elems; tile_r_req_o.tew = tile_ctx_q.access.tew; - // Snapshot a row before issuing its memory requests. - if (tile_req_accept && !mem_spatz_req.op_mem.is_load && mem_spatz_req.op_ope.tss.is_row) begin - tile_rvalid_o = 1'b1; - tile_r_req_o = tile_tss_req; - end - tile_load_rsp_valid = 1'b0; if (tile_store_req_fire) begin @@ -448,17 +479,26 @@ module spatz_doublebw_vlsu tile_ctx_d.row_idx = tile_dim_t'(0); tile_ctx_d.elems = head_tile_elems; tile_ctx_d.is_col = !mem_spatz_req.op_ope.tss.is_row; + tile_ctx_d.word_col = tile_dim_t'(0); tile_ctx_d.bytes = tile_byte_cnt_t'(head_tile_bytes); tile_ctx_d.chunk_sent = tile_chunk_idx_t'(0); tile_ctx_d.packed_data = tile_row_t'(0); - tile_ctx_d.row_buf = tile_row_t'(0); + tile_ctx_d.row_buf = tile_word_t'(0); tile_load_pending_d = {NrMemPorts{1'b0}}; + if (!mem_spatz_req.op_mem.is_load && mem_spatz_req.op_ope.tss.is_row && + (head_tile_bytes <= tile_byte_cnt_t'(TileWordBWidth))) begin + tile_rvalid_o = 1'b1; + tile_r_req_o = tile_tss_req; + end + if (head_tile_elems == tile_count_t'(0)) begin tile_state_d = mem_spatz_req.op_mem.is_load ? Tile_Done : Tile_Idle; end else if (mem_spatz_req.op_mem.is_load) begin tile_state_d = Tile_MemReq; - end else if (mem_spatz_req.op_ope.tss.is_row && tile_rready_i) begin - tile_ctx_d.packed_data = tile_rdata_i; + end else if (mem_spatz_req.op_ope.tss.is_row && + (head_tile_bytes <= tile_byte_cnt_t'(TileWordBWidth)) && + tile_rready_i) begin + tile_ctx_d.packed_data = tile_row_t'(tile_rdata_i); tile_state_d = Tile_MemReq; end else begin tile_state_d = Tile_GatherRead; @@ -470,11 +510,13 @@ module spatz_doublebw_vlsu tile_rvalid_o = 1'b1; if (tile_rready_i) begin if (tile_ctx_q.is_col) begin + tile_count_t local_col; + local_col = tile_count_t'(tile_ctx_q.access.row - tile_req_col); for (int unsigned byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin if (byte_idx < tile_elem_bytes) begin tile_ctx_d.packed_data[ (tile_ctx_q.row_idx*tile_elem_bytes+byte_idx)*8 +: 8] = - tile_rdata_i[(tile_ctx_q.access.row*tile_elem_bytes+byte_idx)*8 +: 8]; + tile_rdata_i[(local_col*tile_elem_bytes+byte_idx)*8 +: 8]; end end @@ -483,8 +525,21 @@ module spatz_doublebw_vlsu else tile_ctx_d.row_idx = tile_ctx_q.row_idx + 1'b1; end else begin - tile_ctx_d.packed_data = tile_rdata_i; - tile_state_d = Tile_MemReq; + for (int unsigned elem = 0; elem < TileEdge; elem++) begin + for (int unsigned byte_idx = 0; byte_idx < AccElemBytes; byte_idx++) begin + if ((elem < tile_req_elems) && (byte_idx < tile_elem_bytes)) begin + tile_ctx_d.packed_data[ + ((tile_req_col+elem)*tile_elem_bytes+byte_idx)*8 +: 8] = + tile_rdata_i[(elem*tile_elem_bytes+byte_idx)*8 +: 8]; + end + end + end + if ((tile_count_t'(tile_ctx_q.word_col) + tile_req_elems) >= tile_ctx_q.elems) begin + tile_ctx_d.word_col = tile_dim_t'(0); + tile_state_d = Tile_MemReq; + end else begin + tile_ctx_d.word_col = tile_dim_t'(tile_ctx_q.word_col + tile_req_elems); + end end end end @@ -521,6 +576,7 @@ module spatz_doublebw_vlsu if ((tile_load_pending_q != '0) && (tile_load_pending_d == '0)) begin if (tile_ctx_q.chunk_sent >= tile_num_chunks) begin tile_ctx_d.row_idx = tile_dim_t'(0); + tile_ctx_d.word_col = tile_dim_t'(0); tile_state_d = tile_ctx_q.is_col ? Tile_CommitRead : Tile_CommitWrite; end else begin tile_state_d = Tile_MemReq; @@ -547,6 +603,9 @@ module spatz_doublebw_vlsu if (tile_ctx_q.is_col && ((tile_count_t'(tile_ctx_q.row_idx) + 1'b1) < tile_ctx_q.elems)) begin tile_ctx_d.row_idx = tile_ctx_q.row_idx + 1'b1; tile_state_d = Tile_CommitRead; + end else if (!tile_ctx_q.is_col && + ((tile_count_t'(tile_ctx_q.word_col) + tile_req_elems) < tile_ctx_q.elems)) begin + tile_ctx_d.word_col = tile_dim_t'(tile_ctx_q.word_col + tile_req_elems); end else if (!vrf_mem_finished && !vrf_rsp_valid) begin tile_load_rsp_valid = 1'b1; tile_state_d = Tile_Idle; @@ -700,6 +759,8 @@ module spatz_doublebw_vlsu ); assign rob_rvalid[intf][fu] = !rob_empty[intf][fu]; assign rob_commit_valid[intf][fu] = rob_rvalid[intf][fu]; + assign rob_id[intf][fu] = '0; + assign rob_rid[intf][fu] = '0; `endif end: gen_rob_intf_fu end: gen_rob_intf @@ -948,36 +1009,29 @@ module spatz_doublebw_vlsu logic [31:0] stride; logic [31:0] offset; - // Pre-shuffling index offset - // Maximum byte offset for eight MAXEW elements with an eight-byte index width. - logic [$clog2(8*8):0] idx_offset; - assign idx_offset = mem_idx_counter_q[intf][fu]; - // Calculate shift amount for address normalization logic [$bits(vew_e)-1:0] log2_num_el_maxew; logic [$bits(vew_e) :0] log2_num_idx_maxew_bytes; logic [2 * MAXEW :0] num_idx_maxew_bytes; + logic [$bits(vlen_t)-1:0] idx_gbyte; + logic [idx_width(N_FU*ELENB)-1:0] word_index; // Number of selected elements contained in one MAXEW element. assign log2_num_el_maxew = MAXEW - mem_data_ew; assign log2_num_idx_maxew_bytes = log2_num_el_maxew + mem_spatz_req.op_mem.ew; // Byte footprint of the indices associated with one MAXEW data element. assign num_idx_maxew_bytes = 1'b1 << log2_num_idx_maxew_bytes; + assign idx_gbyte = + (vlen_t'(fu) << log2_num_idx_maxew_bytes) + + (mem_idx_counter_q[intf][fu] & (num_idx_maxew_bytes - 1)) + + (((mem_idx_counter_q[intf][fu] >> log2_num_idx_maxew_bytes) << + log2_num_idx_maxew_bytes) * N_FU); always_comb begin + word_index = idx_gbyte[idx_width(N_FU*ELENB)-1:0]; stride = mem_is_strided ? mem_spatz_req.rs2 >> mem_data_ew : 'd1; if (mem_is_indexed) begin - // What is the relationship between data and index width? - automatic logic [1:0] data_index_width_diff = int'(mem_spatz_req.vtype.vsew) - int'(mem_spatz_req.op_mem.ew); - - // Pointer to index - automatic logic [idx_width(N_FU*ELENB)-1:0] word_index = - (fu << log2_num_idx_maxew_bytes) + - (idx_offset & (num_idx_maxew_bytes - 1)) + - ((idx_offset >> log2_num_idx_maxew_bytes) << log2_num_idx_maxew_bytes) * - N_FU; - // Index unique case (mem_spatz_req.op_mem.ew) EW_8 : offset = $signed(vrf_rdata_i[intf][1][8 * word_index +: 8]); diff --git a/hw/ip/spatz/src/spatz_pkg.sv.tpl b/hw/ip/spatz/src/spatz_pkg.sv.tpl index cb5eb27c7..4ca53a67f 100644 --- a/hw/ip/spatz/src/spatz_pkg.sv.tpl +++ b/hw/ip/spatz/src/spatz_pkg.sv.tpl @@ -120,16 +120,19 @@ package spatz_pkg; localparam int unsigned NrPhysicalTile = 16; // Tile Dimension localparam int unsigned OPEComputeEdge = 8; - // The ratio of Tile Dimension and OPE FMA array size + // Number of compute-edge groups along one tile edge localparam int unsigned NrAccPerComputeElem = 2; localparam int unsigned TileEdge = OPEComputeEdge * NrAccPerComputeElem; // The controller further limits tk to 32/SEW, or to one for 64-bit accumulation. localparam int unsigned KMAX = (AccElemWidth <= 32) ? (32 / 8) : 1; - // Datapath between VLSU and OPE - localparam int unsigned TileDataWidth = TileEdge*AccElemWidth; + // Maximum storage required for one complete Tile row, in bytes. + localparam int unsigned TileRowBWidth = TileEdge * AccElemBytes; + // Fixed word width of the datapath between VLSU and OPE, in bytes. + localparam int unsigned TileWordBWidth = TileEdge * AccElemBytes; typedef logic [$clog2(TileEdge)-1:0] tile_dim_t; typedef logic [$clog2(TileEdge+1)-1:0] tile_count_t; + typedef logic [$clog2(TileWordBWidth)-1:0] tile_word_elem_idx_t; typedef logic [$clog2(NrPhysicalTile)-1:0] mt_t; typedef struct packed { logic tile_valid; @@ -171,19 +174,22 @@ package spatz_pkg; typedef logic [N_FU*ELENB-1:0] vrf_be_t; typedef logic [N_FU*ELEN-1:0] vrf_data_t; - typedef logic [TileDataWidth-1:0] tile_row_t; + typedef logic [8*TileRowBWidth-1:0] tile_row_t; + typedef logic [8*TileWordBWidth-1:0] tile_word_t; typedef struct packed { mt_t idx; tile_dim_t row; + tile_dim_t col; tile_count_t elems; vew_e tew; } tile_r_req_t; typedef struct packed { mt_t idx; tile_dim_t row; + tile_dim_t col; tile_count_t elems; vew_e tew; - tile_row_t data; + tile_word_t data; } tile_w_req_t; // Instruction ID diff --git a/hw/system/spatz_cluster/Makefile b/hw/system/spatz_cluster/Makefile index cbae48a7b..1cf37489a 100644 --- a/hw/system/spatz_cluster/Makefile +++ b/hw/system/spatz_cluster/Makefile @@ -13,6 +13,7 @@ MKFILE_DIR := $(dir $(MKFILE_PATH)) # Configuration file SPATZ_CLUSTER_CFG ?= spatz_cluster.doublebw.dram.hjson SPATZ_CLUSTER_CFG_PATH := $(SPATZ_CLUSTER_DIR)/cfg/$(SPATZ_CLUSTER_CFG) +VME_TE32 ?= OFF SPATZ_CLUSTER_CFG_DEFINES += -DMEM_DRAM_ORIGIN=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG_PATH)'); print(jstyleson.load(f)['dram']['address'])") SPATZ_CLUSTER_CFG_DEFINES += -DMEM_DRAM_SIZE=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG_PATH)'); print(jstyleson.load(f)['dram']['length'])") SPATZ_CLUSTER_CFG_DEFINES += -DSNRT_BASE_HARTID=$(shell python3 -c "import jstyleson; f = open('$(SPATZ_CLUSTER_CFG_PATH)'); print(jstyleson.load(f)['cluster']['cluster_base_hartid'])") @@ -253,7 +254,7 @@ sw: clean.sw spatz.gendata ## Build SW into sw/build with the LLVM toolchain (including tests) for Questasim simulator sw.vsim: clean.sw bin/spatz_cluster.vsim spatz.gendata mkdir -p sw/build - cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vsim -DBUILD_TESTS=ON -DSPATZ_CLUSTER_CFG=${SPATZ_CLUSTER_CFG} ${SPATZ_CLUSTER_CFG_DEFINES} .. && make -j8 + cd sw/build && ${CMAKE} -DLLVM_PATH=${LLVM_INSTALL_DIR} -DGCC_PATH=${GCC_INSTALL_DIR} -DPYTHON=${PYTHON} -DSNITCH_SIMULATOR=../../../../../hw/system/spatz_cluster/bin/spatz_cluster.vsim -DBUILD_TESTS=ON -DVME_TE32=${VME_TE32} -DSPATZ_CLUSTER_CFG=${SPATZ_CLUSTER_CFG} ${SPATZ_CLUSTER_CFG_DEFINES} .. && make -j8 ## Build SW and run all tests with Questasim simulator sw.test.vsim: sw.vsim diff --git a/hw/system/spatz_cluster/cfg/spatz_cluster.doublebw.dram.hjson b/hw/system/spatz_cluster/cfg/spatz_cluster.doublebw.dram.hjson index 8696fe3d9..a6e8ab0f6 100644 --- a/hw/system/spatz_cluster/cfg/spatz_cluster.doublebw.dram.hjson +++ b/hw/system/spatz_cluster/cfg/spatz_cluster.doublebw.dram.hjson @@ -18,7 +18,7 @@ "cluster_default_axi_user": 1, "axi_cdc_enable": false, "tcdm": { - "size": 512, + "size": 128, "banks": 16, "misalign": true }, @@ -65,24 +65,10 @@ "num_spatz_outstanding_loads": 4, "num_dtlb_entries": 1, "num_itlb_entries": 1 - }, - - // Compute core - { - "isa": "rv32imafd", - "xf16": true, - "xf8": true, - "xfdotp": true, - "xdma": false, - "num_int_outstanding_loads": 1, - "num_int_outstanding_mem": 4, - "num_spatz_outstanding_loads": 4, - "num_dtlb_entries": 1, - "num_itlb_entries": 1 } ], "icache": { - "use_instruction_buffer": true, + "use_instruction_buffer": false, "size": 4, // total instruction cache size in kByte "ways": 2, // number of ways "cacheline": 256 // word size in bits diff --git a/hw/system/spatz_cluster/src/generated/bootrom.sv b/hw/system/spatz_cluster/src/generated/bootrom.sv index c1cfec4d8..907d052f2 100644 --- a/hw/system/spatz_cluster/src/generated/bootrom.sv +++ b/hw/system/spatz_cluster/src/generated/bootrom.sv @@ -20,8 +20,8 @@ module bootrom #( localparam int AddrBits = RomSize > 1 ? $clog2(RomSize) : 1; const logic [RomSize-1:0][DataWidth-1:0] mem = { - 512'h00000000_00000000_00000000_00000000_00000000_00000000_00000000_000010d8_00000001_00000000_00000000_80000000_00000000_00000000_00000000_00080000, - 512'h00000000_00100000_00000000_00000000_00000000_00000002_00000000_00001000_ffdff06f_10500073_00038067_0003a383_00038393_05838393_01c383b3_0205ae03, + 512'h00000000_00000000_00000000_00000000_00000000_00000000_00000000_000010d8_00000001_00000000_00000000_80000000_00000000_00000000_00000000_00020000, + 512'h00000000_00100000_00000000_00000000_00000000_00000001_00000000_00001000_ffdff06f_10500073_00038067_0003a383_00038393_05838393_01c383b3_0205ae03, 512'h0185a383_10500073_30461073_00666633_00080337_30402673_30461073_00866613_30402673_30431073_80030313_00001337_05858593_00000597_30531073_05c30313, 512'h00000317_00000f93_00000f13_00000e93_00000e13_00000d93_00000d13_00000c93_00000c13_00000b93_00000b13_00000a93_00000a13_00000993_00000913_00000893, 512'h00000813_00000793_00000713_00000693_00000613_00000593_00000513_00000493_00000413_00000393_00000313_00000293_00000213_00000193_00000113_00000093 diff --git a/hw/system/spatz_cluster/src/generated/spatz_cluster_wrapper.sv b/hw/system/spatz_cluster/src/generated/spatz_cluster_wrapper.sv index 3d1b4e5e7..06523c094 100644 --- a/hw/system/spatz_cluster/src/generated/spatz_cluster_wrapper.sv +++ b/hw/system/spatz_cluster/src/generated/spatz_cluster_wrapper.sv @@ -50,14 +50,14 @@ package spatz_cluster_pkg; // Spatz Cluster // //////////////////// - localparam int unsigned NumCores = 2; + localparam int unsigned NumCores = 1; localparam int unsigned ICacheLineWidth = 256; localparam int unsigned ICacheLineCount = 64; localparam int unsigned ICacheWays = 2; localparam int unsigned TCDMStartAddr = 32'h100000; - localparam int unsigned TCDMSize = 32'h80000; + localparam int unsigned TCDMSize = 32'h20000; localparam logic AddrMisalign = 1'b1; // 0-aligned, 1-misalign localparam int unsigned PeriStartAddr = TCDMStartAddr + TCDMSize; @@ -77,68 +77,6 @@ package spatz_cluster_pkg; }; localparam fpnew_pkg::fpu_implementation_t FPUImplementation [NumCores] = '{ - '{ - PipeRegs: // FMA Block - '{ - '{ 1, // FP32 - 2, // FP64 - 0, // FP16 - 0, // FP8 - 0, // FP16alt - 0 // FP8alt - }, - '{1, 1, 1, 1, 1, 1}, // DIVSQRT - '{1, - 1, - 1, - 1, - 1, - 1}, // NONCOMP - '{2, - 2, - 2, - 2, - 2, - 2}, // CONV - '{2, - 2, - 2, - 2, - 2, - 2} // DOTP - }, - UnitTypes: '{'{fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED}, // FMA - '{fpnew_pkg::DISABLED, - fpnew_pkg::DISABLED, - fpnew_pkg::DISABLED, - fpnew_pkg::DISABLED, - fpnew_pkg::DISABLED, - fpnew_pkg::DISABLED}, // DIVSQRT - '{fpnew_pkg::PARALLEL, - fpnew_pkg::PARALLEL, - fpnew_pkg::PARALLEL, - fpnew_pkg::PARALLEL, - fpnew_pkg::PARALLEL, - fpnew_pkg::PARALLEL}, // NONCOMP - '{fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED}, // CONV - '{fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED, - fpnew_pkg::MERGED}}, // DOTP - PipeConfig: fpnew_pkg::BEFORE - }, '{ PipeRegs: // FMA Block '{ @@ -238,9 +176,9 @@ module spatz_cluster_wrapper input axi_out_resp_t axi_out_resp_i ); - localparam int unsigned NumIntOutstandingLoads [NumCores] = '{1, 1}; - localparam int unsigned NumIntOutstandingMem [NumCores] = '{4, 4}; - localparam int unsigned NumSpatzOutstandingLoads [NumCores] = '{4, 4}; + localparam int unsigned NumIntOutstandingLoads [NumCores] = '{1}; + localparam int unsigned NumIntOutstandingMem [NumCores] = '{4}; + localparam int unsigned NumSpatzOutstandingLoads [NumCores] = '{4}; localparam int unsigned NumSpatzFPUs [NumCores] = '{default: 4}; localparam int unsigned NumSpatzIPUs [NumCores] = '{default: 1}; localparam int unsigned NumSpatzTCDMPorts [NumCores] = '{default: 8}; @@ -289,10 +227,11 @@ module spatz_cluster_wrapper .NarrowAXIDataWidth (SpatzNarrowAxiDataWidth), .BootAddr (32'h1000), .ClusterPeriphSize (64), - .NrCores (2), - .TCDMDepth (4096), + .NrCores (1), + .TCDMDepth (1024), .TCDMSize (TCDMSize), .NrBanks (16), + .UseInstructionBuffer (0), .ICacheLineWidth (spatz_cluster_pkg::ICacheLineWidth), .ICacheLineCount (spatz_cluster_pkg::ICacheLineCount), .ICacheWays (spatz_cluster_pkg::ICacheWays), @@ -309,7 +248,7 @@ module spatz_cluster_wrapper .axi_in_resp_t (axi_in_resp_t), .axi_out_req_t (spatz_axi_iwc_out_req_t), .axi_out_resp_t (spatz_axi_iwc_out_resp_t), - .Xdma (2'b01), + .Xdma (1'b1), .DMAAxiReqFifoDepth (3), .DMAReqFifoDepth (3), .RegisterOffloadRsp (1), diff --git a/hw/system/spatz_cluster/src/spatz_cluster_wrapper.sv.tpl b/hw/system/spatz_cluster/src/spatz_cluster_wrapper.sv.tpl index 7c8941ef1..8919c9da9 100644 --- a/hw/system/spatz_cluster/src/spatz_cluster_wrapper.sv.tpl +++ b/hw/system/spatz_cluster/src/spatz_cluster_wrapper.sv.tpl @@ -508,6 +508,7 @@ module ${cfg['name']}_wrapper .TCDMDepth (${cfg['tcdm']['depth']}), .TCDMSize (TCDMSize), .NrBanks (${cfg['tcdm']['banks']}), + .UseInstructionBuffer (${int(cfg['icache']['use_instruction_buffer'])}), .ICacheLineWidth (${cfg['pkg_name']}::ICacheLineWidth), .ICacheLineCount (${cfg['pkg_name']}::ICacheLineCount), .ICacheWays (${cfg['pkg_name']}::ICacheWays), diff --git a/hw/system/spatz_cluster/sw/CMakeLists.txt b/hw/system/spatz_cluster/sw/CMakeLists.txt index 16fc7e393..f704edf91 100644 --- a/hw/system/spatz_cluster/sw/CMakeLists.txt +++ b/hw/system/spatz_cluster/sw/CMakeLists.txt @@ -16,5 +16,5 @@ include(SnitchUtilities) enable_testing() add_subdirectory(${SNITCH_SOFTWARE_DIR}/snRuntime snRuntime) add_subdirectory(${SNITCH_SOFTWARE_DIR}/riscvTests riscvTests) -add_subdirectory(${SNITCH_SOFTWARE_DIR}/spatzBenchmarks spatzBenchmarks) +# add_subdirectory(${SNITCH_SOFTWARE_DIR}/spatzBenchmarks spatzBenchmarks) add_subdirectory(${SNITCH_SOFTWARE_DIR}/vme vme) \ No newline at end of file diff --git a/hw/system/spatz_cluster/test/bootrom.bin b/hw/system/spatz_cluster/test/bootrom.bin index d9cbcae45..9ce12bfe8 100755 Binary files a/hw/system/spatz_cluster/test/bootrom.bin and b/hw/system/spatz_cluster/test/bootrom.bin differ diff --git a/hw/system/spatz_cluster/test/bootrom.dump b/hw/system/spatz_cluster/test/bootrom.dump index 1fd43d359..5c20d026f 100644 --- a/hw/system/spatz_cluster/test/bootrom.dump +++ b/hw/system/spatz_cluster/test/bootrom.dump @@ -71,13 +71,13 @@ Disassembly of section .rodata: 10e2: 0000 unimp 10e4: 0000 unimp 10e6: 0000 unimp - 10e8: 0002 0x2 + 10e8: 0001 nop ... 10fa: 0010 addi a2,sp,0 10fc: 0000 unimp 10fe: 0000 unimp 1100: 0000 unimp - 1102: 0008 addi a0,sp,0 + 1102: 0002 0x2 ... 1110: 0000 unimp 1112: 8000 0x8000 diff --git a/hw/system/spatz_cluster/test/bootrom.elf b/hw/system/spatz_cluster/test/bootrom.elf index 2a2b9b895..6853ec7f3 100755 Binary files a/hw/system/spatz_cluster/test/bootrom.elf and b/hw/system/spatz_cluster/test/bootrom.elf differ diff --git a/sw/.gitignore b/sw/.gitignore index fd7ba073a..dbb8fdc51 100644 --- a/sw/.gitignore +++ b/sw/.gitignore @@ -23,3 +23,6 @@ toolchain/ *.a *.la *.lo + +# Generated VME application data headers +/vme/apps/**/data/data_*_*_*.h diff --git a/sw/riscvTests/isa/macros/vector/vector_matrix_macros.h b/sw/riscvTests/isa/macros/vector/vector_matrix_macros.h index c4afdc789..ec8bf9559 100644 --- a/sw/riscvTests/isa/macros/vector/vector_matrix_macros.h +++ b/sw/riscvTests/isa/macros/vector/vector_matrix_macros.h @@ -9,70 +9,88 @@ #ifndef __VECTOR_MATRIX_MACROS_H__ #define __VECTOR_MATRIX_MACROS_H__ -#include "dataset.h" -#include "encoding.h" #include -#include -// ------------------------------------------------------- -// TSS (Tile Subset Specifier) — VME Spec §1.5 -// bits[30:27] = tile index (0–15) -// bits[26:24] = pattern (0 = row, 1 = col) -// bits[23:0] = row / col index -// ------------------------------------------------------- -#define TSS_ROW(tile, row) \ +// Tile subset specifier: tile[30:27], pattern[26:24], index[23:0]. +#define TSS(tile, pattern, index) \ (((uintptr_t)((tile) & 0xF) << 27) | \ - (uintptr_t)((row) & 0xFFFFFF)) + ((uintptr_t)((pattern) & 0x7) << 24) | \ + (uintptr_t)((index) & 0xFFFFFF)) +#define TSS_ROW(tile, row) TSS(tile, 0, row) +#define TSS_COL(tile, col) TSS(tile, 1, col) -#define TSS_COL(tile, col) \ - (((uintptr_t)((tile) & 0xF) << 27) | \ - ((uintptr_t)1 << 24) | \ - (uintptr_t)((col) & 0xFFFFFF)) - -// ------------------------------------------------------- -// mtype CSR (0xC23) field extractors -// ------------------------------------------------------- +// mtype and vtype constructors and field extractors. +#define MTYPE(mtwiden, tk, tm) \ + ((uintptr_t)(((mtwiden) & 0x3) | (((tk) & 0x7) << 5) | \ + (((tm) & 0x3FFF) << 10))) +#define VTYPE(vsew, vlmul, vta, vma, altfmt) \ + ((uintptr_t)(((vlmul) & 0x7) | (((vsew) & 0x7) << 3) | \ + (((vta) & 0x1) << 6) | (((vma) & 0x1) << 7) | \ + (((altfmt) & 0x1) << 8))) #define MTYPE_MTWIDEN(mtype) (((mtype) >> 0) & 0x3) #define MTYPE_TK(mtype) (((mtype) >> 5) & 0x7) #define MTYPE_TM(mtype) (((mtype) >> 10) & 0x3FFF) -#define KMAX 4 +// Zvt v0.4 KMAX dictionary, keyed by SEW and TWIDEN (not their encodings). +#define KMAX_8_1 4 +#define KMAX_8_2 4 +#define KMAX_8_4 4 +#define KMAX_16_1 2 +#define KMAX_16_2 2 +#define KMAX_16_4 2 +#define KMAX_32_1 1 +#define KMAX_32_2 1 +#define KMAX_64_1 1 +#define KMAX_LOOKUP(sew, twiden) KMAX_##sew##_##twiden +#define KMAX(sew, twiden) KMAX_LOOKUP(sew, twiden) -// ------------------------------------------------------- -// Expected mtype / vtype values (for msetmtype* tests) -// -// mtype = uimm5 & 0x1F -// vtype = (vsew << 3) | (mtwiden != 0 ? 0xC0 : 0x00) -// where vsew = uimm3, mtwiden = uimm5 & 0x3 -// ------------------------------------------------------- -#define MTYPE_EXP(uimm5) \ - ((uintptr_t)((uimm5) & 0x1Fu)) +#ifdef VME_DIMENSION_TEST +// Independent configuration model for the dimension ISA tests. +// The current Spatz package sets TileEdge to 8 * 2 elements. +#define VME_TEST_TE 16 +typedef struct { + unsigned sew, twiden, vsew, mtwiden, kmax; +} vme_test_config_t; -#define VTYPE_EXP(uimm5, uimm3) \ - ((uintptr_t)(((uimm3) & 0x7u) << 3) | \ - (((uimm5) & 0x3u) ? (uintptr_t)0xC0u : (uintptr_t)0u)) +#define VME_CONFIG(sew, twiden, vsew, mtwiden) \ + {sew, twiden, vsew, mtwiden, KMAX(sew, twiden)} +static const vme_test_config_t vme_test_configs[] = { + VME_CONFIG(8, 1, 0, 1), VME_CONFIG(8, 2, 0, 2), + VME_CONFIG(8, 4, 0, 3), VME_CONFIG(16, 1, 1, 1), + VME_CONFIG(16, 2, 1, 2), VME_CONFIG(32, 1, 2, 1), +}; -#define VME_VTYPE(vsew_enc) \ - ((uintptr_t)(((vsew_enc) & 0x7u) << 3) | 0xC0u) +static inline uintptr_t vme_test_min(uintptr_t a, uintptr_t b) { + return a < b ? a : b; +} -#define MTYPE_VAL(mtwiden) \ - ((uintptr_t)((mtwiden) & 0x3u)) +static inline uintptr_t vme_test_lmul(const vme_test_config_t *cfg, + uintptr_t vlenb) { + uintptr_t ete = cfg->sew * cfg->twiden < 64 ? VME_TEST_TE : VME_TEST_TE / 2; + uintptr_t eve = vlenb * 8 / cfg->sew; + return vme_test_min((ete + eve - 1) / eve, + vme_test_min(8 / cfg->kmax, 8 / cfg->twiden)); +} -// ------------------------------------------------------- -// enable_vme — set mstatus.MS = Initial (01b) -// -// mstatus.MS = bits[30:29] (MSTATUS_MS = 0x60000000) -// Initial state = 01b → set bit 29 only -// Same pattern as enable_vec() / enable_fp(). -// ------------------------------------------------------- -#ifndef MSTATUS_MS -#define MSTATUS_MS 0x60000000UL -#endif +static inline uintptr_t vme_test_capacity(const vme_test_config_t *cfg, + uintptr_t vlenb) { + uintptr_t ete = cfg->sew * cfg->twiden < 64 ? VME_TEST_TE : VME_TEST_TE / 2; + return vme_test_min(ete, vme_test_lmul(cfg, vlenb) * (vlenb * 8 / cfg->sew)); +} + +static inline uintptr_t vme_test_vtype(const vme_test_config_t *cfg, + uintptr_t vlenb) { + uintptr_t lmul = vme_test_lmul(cfg, vlenb), enc = 0; + while (lmul > 1) { lmul >>= 1; ++enc; } + return VTYPE(cfg->vsew, enc, 1, 1, 0); +} -#define enable_vme() \ - do { \ - asm volatile("csrs mstatus, %[bits]" \ - :: [bits]"r"(MSTATUS_MS & (MSTATUS_MS >> 1))); \ - } while (0) +// Seed all dimensions without relying on another dimension-setting instruction. +static inline void vme_test_setup(const vme_test_config_t *cfg) { + uintptr_t mtype = MTYPE(cfg->mtwiden, 1, 1); + uintptr_t vtype = VTYPE(cfg->vsew, 0, 0, 0, 0); + asm volatile("msetmtype %0, %1" :: "r"(mtype), "r"(vtype) : "memory"); +} +#endif #endif // __VECTOR_MATRIX_MACROS_H__ diff --git a/sw/toolchain/llvm-project.version b/sw/toolchain/llvm-project.version index bfab8916a..a7e8223d8 100644 --- a/sw/toolchain/llvm-project.version +++ b/sw/toolchain/llvm-project.version @@ -1 +1 @@ -d1227fa0c99d1a632a868b4e8afa0fff00fc7ed9 +f19be69f3428e21e7adcee9ddb7347fec255ed83 diff --git a/sw/toolchain/riscv-opcodes.version b/sw/toolchain/riscv-opcodes.version index 76b3f7e59..f7b980458 100644 --- a/sw/toolchain/riscv-opcodes.version +++ b/sw/toolchain/riscv-opcodes.version @@ -1 +1 @@ -b98a8cc20a068fec74b18392e29d5d59f7c4f8b7 \ No newline at end of file +78e7e87f20ddae7e923e3916cba932401982bb7a diff --git a/sw/vme/CMakeLists.txt b/sw/vme/CMakeLists.txt index ee0515e74..a2e2cb0e1 100644 --- a/sw/vme/CMakeLists.txt +++ b/sw/vme/CMakeLists.txt @@ -26,5 +26,5 @@ include_directories(${SNRUNTIME_INCLUDE_DIRS}) include_directories(../riscvTests/isa/macros/vector) # Tests -# add_subdirectory(isaTests) +add_subdirectory(isaTests) add_subdirectory(apps) \ No newline at end of file diff --git a/sw/vme/apps/CMakeLists.txt b/sw/vme/apps/CMakeLists.txt index e550f76b7..0b910749d 100644 --- a/sw/vme/apps/CMakeLists.txt +++ b/sw/vme/apps/CMakeLists.txt @@ -6,6 +6,13 @@ cmake_minimum_required(VERSION 3.13) +# VME binaries require a configured TCDM capacity for buffer sizing. +if(NOT DEFINED SNRT_TCDM_SIZE OR NOT SNRT_TCDM_SIZE GREATER 0) + message(STATUS "Skipping VME apps: SNRT_TCDM_SIZE is not configured") + return() +endif() +add_compile_definitions(SNRT_TCDM_SIZE=${SNRT_TCDM_SIZE}) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/kernel) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/data) @@ -13,8 +20,22 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/data) enable_testing() set(SNITCH_TEST_PREFIX vme-) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/vme/apps") + +option(VME_TE32 "Build VME kernels for a TE=32 hardware configuration" OFF) +if(VME_TE32) + set(VME_TILE_VARIANT te32) +else() + set(VME_TILE_VARIANT te16) +endif() +message(STATUS "Building VME ${VME_TILE_VARIANT} kernels") add_subdirectory(tew32/fp32-matmul-vtse fp32-matmul-vtse) add_subdirectory(tew32/fp32-matmul-vse fp32-matmul-vse) add_subdirectory(tew32/fp16-matmul-vtse fp16-matmul-vtse) -# add_subdirectory(tew32/fp16-matmul-vse fp16-matmul-vse) +add_subdirectory(tew32/fp16-matmul-vse fp16-matmul-vse) +add_subdirectory(tew32/fp16-gemm-unit fp16-gemm-unit) +add_subdirectory(tew32/fp16-gemm-alpha fp16-gemm-alpha) +add_subdirectory(tew32/fp16-gemm-beta fp16-gemm-beta) + +# add_subdirectory(tew32/fp16-vq-gemm fp16-vq-gemm) diff --git a/sw/vme/apps/include/ope_transformer.h b/sw/vme/apps/include/ope_transformer.h new file mode 100644 index 000000000..408e72702 --- /dev/null +++ b/sw/vme/apps/include/ope_transformer.h @@ -0,0 +1,133 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +// Adapted from baseline_spatz/sw/opopeBenchmarks/include/opope_transformer.h. +// Shared OPE application vector math. No divider, sqrt, vfrec7 or vfrsqrt7. +// Instruction-string macros compose inside ONE asm volatile block, allowing +// the caller to interleave loads and compute without function calls. +// Register arguments are string literals for disjoint aligned LMUL groups. +// The caller must set e32/VL/LMUL and list every physical register clobbered. +// Operand bundles below provide the required named scalar input operands. +#define OPE_EXP_INPUTS \ + [ope_lo] "f"(-16.0f), [ope_hi] "f"(20.0f), \ + [ope_scale] "f"(0.015625f), [ope_c5] "f"(0.0083333333f), \ + [ope_c4] "f"(0.0416666667f), [ope_c3] "f"(0.1666666667f), \ + [ope_c2] "f"(0.5f), [ope_one] "f"(1.0f) +#define OPE_RECIP_INPUTS \ + [ope_seed] "r"(0x7ef311c7u), [ope_two] "f"(2.0f) +#define OPE_RSQRT_INPUTS \ + [ope_rsqrt_seed] "r"(0x5f3759dfu), \ + [ope_half] "f"(0.5f), [ope_three_halves] "f"(1.5f) +#define OPE_RELU_INPUTS [ope_zero] "f"(0.0f) + +#define OPE_EXP_ASM(x, t) \ + "vfmax.vf " x ", " x ", %[ope_lo]\n" \ + "vfmin.vf " x ", " x ", %[ope_hi]\n" \ + "vfmul.vf " x ", " x ", %[ope_scale]\n" \ + "vfmv.v.f " t ", %[ope_c5]\n" \ + "vfmul.vv " t ", " t ", " x "\n" \ + "vfadd.vf " t ", " t ", %[ope_c4]\n" \ + "vfmul.vv " t ", " t ", " x "\n" \ + "vfadd.vf " t ", " t ", %[ope_c3]\n" \ + "vfmul.vv " t ", " t ", " x "\n" \ + "vfadd.vf " t ", " t ", %[ope_c2]\n" \ + "vfmul.vv " t ", " t ", " x "\n" \ + "vfadd.vf " t ", " t ", %[ope_one]\n" \ + "vfmul.vv " t ", " t ", " x "\n" \ + "vfadd.vf " t ", " t ", %[ope_one]\n" \ + "vfmul.vv " x ", " t ", " t "\n" \ + "vfmul.vv " x ", " x ", " x "\n" \ + "vfmul.vv " x ", " x ", " x "\n" \ + "vfmul.vv " x ", " x ", " x "\n" \ + "vfmul.vv " x ", " x ", " x "\n" \ + "vfmul.vv " x ", " x ", " x "\n" + +#define OPE_EXP_PAIR_ASM(x0, x1, t0, t1) \ + "vfmax.vf " x0 ", " x0 ", %[ope_lo]\n" \ + "vfmax.vf " x1 ", " x1 ", %[ope_lo]\n" \ + "vfmin.vf " x0 ", " x0 ", %[ope_hi]\n" \ + "vfmin.vf " x1 ", " x1 ", %[ope_hi]\n" \ + "vfmul.vf " x0 ", " x0 ", %[ope_scale]\n" \ + "vfmul.vf " x1 ", " x1 ", %[ope_scale]\n" \ + "vfmv.v.f " t0 ", %[ope_c5]\n" \ + "vfmv.v.f " t1 ", %[ope_c5]\n" \ + "vfmul.vv " t0 ", " t0 ", " x0 "\n" \ + "vfmul.vv " t1 ", " t1 ", " x1 "\n" \ + "vfadd.vf " t0 ", " t0 ", %[ope_c4]\n" \ + "vfadd.vf " t1 ", " t1 ", %[ope_c4]\n" \ + "vfmul.vv " t0 ", " t0 ", " x0 "\n" \ + "vfmul.vv " t1 ", " t1 ", " x1 "\n" \ + "vfadd.vf " t0 ", " t0 ", %[ope_c3]\n" \ + "vfadd.vf " t1 ", " t1 ", %[ope_c3]\n" \ + "vfmul.vv " t0 ", " t0 ", " x0 "\n" \ + "vfmul.vv " t1 ", " t1 ", " x1 "\n" \ + "vfadd.vf " t0 ", " t0 ", %[ope_c2]\n" \ + "vfadd.vf " t1 ", " t1 ", %[ope_c2]\n" \ + "vfmul.vv " t0 ", " t0 ", " x0 "\n" \ + "vfmul.vv " t1 ", " t1 ", " x1 "\n" \ + "vfadd.vf " t0 ", " t0 ", %[ope_one]\n" \ + "vfadd.vf " t1 ", " t1 ", %[ope_one]\n" \ + "vfmul.vv " t0 ", " t0 ", " x0 "\n" \ + "vfmul.vv " t1 ", " t1 ", " x1 "\n" \ + "vfadd.vf " t0 ", " t0 ", %[ope_one]\n" \ + "vfadd.vf " t1 ", " t1 ", %[ope_one]\n" \ + "vfmul.vv " x0 ", " t0 ", " t0 "\n" \ + "vfmul.vv " x1 ", " t1 ", " t1 "\n" \ + "vfmul.vv " x0 ", " x0 ", " x0 "\n" \ + "vfmul.vv " x1 ", " x1 ", " x1 "\n" \ + "vfmul.vv " x0 ", " x0 ", " x0 "\n" \ + "vfmul.vv " x1 ", " x1 ", " x1 "\n" \ + "vfmul.vv " x0 ", " x0 ", " x0 "\n" \ + "vfmul.vv " x1 ", " x1 ", " x1 "\n" \ + "vfmul.vv " x0 ", " x0 ", " x0 "\n" \ + "vfmul.vv " x1 ", " x1 ", " x1 "\n" \ + "vfmul.vv " x0 ", " x0 ", " x0 "\n" \ + "vfmul.vv " x1 ", " x1 ", " x1 "\n" + +#define OPE_NR_RECIP_STEP_ASM(x, r, t) \ + "vfmul.vv " t ", " x ", " r "\n" \ + "vfrsub.vf " t ", " t ", %[ope_two]\n" \ + "vfmul.vv " r ", " r ", " t "\n" + +#define OPE_RECIP_ASM(x, r, t) \ + "vrsub.vx " r ", " x ", %[ope_seed]\n" \ + OPE_NR_RECIP_STEP_ASM(x, r, t) \ + OPE_NR_RECIP_STEP_ASM(x, r, t) \ + OPE_NR_RECIP_STEP_ASM(x, r, t) + +#define OPE_NR_RSQRT_STEP_ASM(r, t, half_x) \ + "vfmul.vv " t ", " r ", " r "\n" \ + "vfmul.vv " t ", " t ", " half_x "\n" \ + "vfrsub.vf " t ", " t ", %[ope_three_halves]\n" \ + "vfmul.vv " r ", " r ", " t "\n" + +#define OPE_RSQRT_ASM(x, r, t, half_x) \ + "vsrl.vi " r ", " x ", 1\n" \ + "vrsub.vx " r ", " r ", %[ope_rsqrt_seed]\n" \ + "vfmul.vf " half_x ", " x ", %[ope_half]\n" \ + OPE_NR_RSQRT_STEP_ASM(r, t, half_x) \ + OPE_NR_RSQRT_STEP_ASM(r, t, half_x) \ + OPE_NR_RSQRT_STEP_ASM(r, t, half_x) + +#define OPE_RELU_ASM(x) "vfmax.vf " x ", " x ", %[ope_zero]\n" +#define OPE_NORMALIZE_ASM(x, inv) "vfmul.vv " x ", " x ", " inv "\n" +#define OPE_LOAD_ADVANCE_ASM(v, p, bytes) \ + "vle32.v " v ", (" p ")\n" "addi " p ", " p ", " bytes "\n" +#define OPE_STORE_ADVANCE_ASM(v, p, bytes) \ + "vse32.v " v ", (" p ")\n" "addi " p ", " p ", " bytes "\n" + +// O-POPE tanh-form GELU, using the same clamped exp and 3-step reciprocal. +// All five vector register arguments must be disjoint. +#define OPE_GELU_INPUTS OPE_EXP_INPUTS, OPE_RECIP_INPUTS, \ + [ope_gelu_c] "f"(0.044715f), [ope_gelu_scale] "f"(-1.5957691216f) +#define OPE_GELU_ASM(x, e, t, r, q) \ + "vfmul.vv " e ", " x ", " x "\n" \ + "vfmul.vf " e ", " e ", %[ope_gelu_c]\n" \ + "vfadd.vf " e ", " e ", %[ope_one]\n" \ + "vfmul.vv " e ", " e ", " x "\n" \ + "vfmul.vf " e ", " e ", %[ope_gelu_scale]\n" \ + OPE_EXP_ASM(e, t) \ + "vfadd.vf " e ", " e ", %[ope_one]\n" \ + OPE_RECIP_ASM(e, r, q) \ + "vfmul.vv " x ", " x ", " r "\n" diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/data/data_64_64_128.h b/sw/vme/apps/tew16/fp16-matmul-vtse/data/data_64_64_128.h deleted file mode 100644 index 0fc14e74f..000000000 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/data/data_64_64_128.h +++ /dev/null @@ -1,15 +0,0 @@ -// Auto-generated by tew16/fp16-matmul-vtse/script/gen_data.py. -#include "layer.h" - -static const vme_matmul_layer matmul_l = { - .M = 64, .N = 64, .K = 128, -}; - -static const float Cref_dram[4096] - __attribute__((section(".dram"), aligned(64))) = {-6.66796875f, -9.671875f, -27.1875f, 2.744140625f, 6.54296875f, 20.453125f, -1.8349609375f, 8.625f, -15.6796875f, -3.048828125f, -6.2578125f, -1.2275390625f, 7.390625f, 2.03125f, -3.2890625f, 2.8515625f, -8.125f, -4.47265625f, -7.78515625f, 4.265625f, -7.98046875f, 3.017578125f, 7.4296875f, 20.125f, 18.4375f, -4.12109375f, 4.359375f, -9.84375f, 13.046875f, -5.92578125f, -8.0390625f, -16.34375f, -5.17578125f, 1.97265625f, -1.095703125f, 5.50390625f, 15.140625f, 5.0f, -11.84375f, 12.109375f, -6.4140625f, 4.83984375f, 3.583984375f, -11.5234375f, -2.978515625f, 1.5859375f, -1.294921875f, 6.125f, -6.97265625f, -17.296875f, -14.0625f, 15.375f, 5.60546875f, 12.40625f, 1.7841796875f, 17.375f, 6.82421875f, -16.515625f, 19.6875f, -11.609375f, 28.140625f, -8.921875f, -13.1796875f, -14.2265625f, -1.7705078125f, -19.796875f, 10.3515625f, 21.75f, 2.26171875f, -2.458984375f, 18.171875f, -0.3369140625f, 1.90625f, 17.34375f, -4.4765625f, 3.466796875f, 6.234375f, 7.2890625f, -0.166259765625f, 1.9580078125f, -18.65625f, -4.109375f, -8.4609375f, -0.272216796875f, 3.005859375f, -6.55859375f, 14.34375f, -2.4375f, -8.8671875f, 0.05377197265625f, -20.359375f, -0.982421875f, -3.505859375f, -1.353515625f, 2.384765625f, 12.40625f, 19.5625f, -7.0390625f, 10.21875f, -17.1875f, -11.65625f, -0.477783203125f, 14.6484375f, -4.22265625f, 18.9375f, 12.984375f, 19.0625f, -1.9013671875f, 2.20703125f, 2.123046875f, 4.98828125f, -2.52734375f, -12.609375f, -20.03125f, -8.6484375f, 16.671875f, 7.48046875f, 17.640625f, 11.4375f, 14.6015625f, 16.921875f, -15.671875f, -10.03125f, -17.453125f, -8.3125f, -4.3828125f, -6.1640625f, 12.234375f, 8.953125f, -11.8828125f, 6.3671875f, 6.59765625f, -22.953125f, -6.46875f, 23.78125f, 15.0625f, -24.5625f, 10.125f, -7.94921875f, 5.58203125f, 13.234375f, 3.853515625f, -16.203125f, 7.59375f, -16.3125f, -2.9453125f, 7.70703125f, -5.453125f, -5.8046875f, -14.984375f, -16.703125f, -14.484375f, -1.3583984375f, -10.515625f, 8.015625f, 3.66015625f, -16.15625f, -6.18359375f, 10.5859375f, -1.400390625f, -4.35546875f, -17.921875f, -12.1484375f, 25.59375f, 4.97265625f, -4.3671875f, -20.40625f, 0.88916015625f, -13.046875f, -15.3984375f, 23.34375f, -4.140625f, 6.00390625f, -1.2685546875f, -0.529296875f, 1.634765625f, -6.5390625f, -4.65625f, -12.2265625f, -8.3203125f, -6.08984375f, 6.609375f, -13.046875f, 4.2890625f, -1.2099609375f, 0.358642578125f, 6.109375f, 7.99609375f, 21.6875f, -10.6953125f, -2.896484375f, -3.634765625f, -3.021484375f, -9.0625f, -5.203125f, 30.546875f, -12.9609375f, 14.765625f, 3.046875f, 10.4609375f, 11.765625f, -8.921875f, -9.15625f, 32.4375f, 0.1717529296875f, 6.79296875f, -1.439453125f, 4.16796875f, 20.640625f, -6.61328125f, -3.71484375f, 4.9921875f, -1.66015625f, 10.734375f, 9.8203125f, -14.203125f, 5.6484375f, -3.95703125f, 23.625f, -13.09375f, 3.1015625f, 4.9609375f, 3.14453125f, -0.6865234375f, 13.21875f, 16.96875f, 17.484375f, -2.248046875f, 2.400390625f, -11.234375f, 4.609375f, 27.546875f, 2.740234375f, 9.1015625f, 23.921875f, 19.640625f, -4.671875f, 12.6796875f, 14.53125f, 27.84375f, 8.6171875f, 9.359375f, -3.048828125f, 18.5625f, -6.02734375f, -24.9375f, 6.2734375f, -6.6015625f, -10.3359375f, -26.0625f, -8.984375f, 3.37890625f, -10.1015625f, -0.75390625f, 23.046875f, 6.34765625f, 15.984375f, 13.8203125f, -2.71875f, -4.9609375f, -8.390625f, 11.21875f, 13.8125f, -4.015625f, -1.2255859375f, -5.71484375f, -13.671875f, 2.41796875f, 14.8984375f, 4.70703125f, -23.109375f, 1.4521484375f, 13.1015625f, -14.390625f, -6.71875f, -2.634765625f, -8.46875f, 4.1796875f, 5.86328125f, 9.6484375f, 0.59423828125f, 17.015625f, 2.94140625f, -16.265625f, 4.86328125f, -6.65234375f, -0.34716796875f, 19.71875f, -8.5546875f, 22.5f, 15.15625f, -0.064453125f, -9.4609375f, 3.98046875f, -1.248046875f, -3.787109375f, -2.32421875f, -0.86376953125f, 13.28125f, 6.88671875f, 9.8125f, 3.841796875f, -14.2421875f, 15.1484375f, 10.8203125f, -4.05859375f, -2.00390625f, 19.921875f, -12.109375f, -15.4375f, -8.640625f, -2.470703125f, 19.640625f, -20.25f, -12.296875f, 6.578125f, 18.484375f, 5.0390625f, -5.19140625f, 10.9609375f, -17.078125f, 21.0f, -21.875f, -21.0625f, 2.185546875f, 17.125f, -10.0625f, -17.578125f, 10.5625f, 22.0625f, 13.53125f, -8.9921875f, 3.005859375f, -7.171875f, -6.10546875f, 8.9296875f, -0.51904296875f, -5.171875f, -12.515625f, 11.203125f, 4.0390625f, 3.58984375f, 12.34375f, 10.234375f, 11.15625f, 3.0078125f, -14.453125f, -0.5830078125f, 11.1953125f, 20.390625f, -4.09375f, -5.15625f, -8.859375f, 14.1015625f, 24.6875f, -3.763671875f, -13.8984375f, -3.6640625f, 23.8125f, 17.984375f, 2.484375f, -7.9921875f, -1.466796875f, 10.1015625f, -14.21875f, -6.76171875f, 3.716796875f, 16.046875f, -2.59765625f, 2.240234375f, -36.0625f, 18.1875f, -4.953125f, -8.1328125f, 12.34375f, -2.884765625f, 4.171875f, 9.078125f, 20.171875f, 15.65625f, -20.296875f, 9.9296875f, 10.8203125f, 5.8046875f, 5.6796875f, -4.8828125f, 5.4375f, -5.28125f, 15.8671875f, 18.0625f, 14.6484375f, -5.046875f, -9.3046875f, -7.71875f, 3.197265625f, 7.05859375f, 5.5625f, -22.796875f, -5.62109375f, 17.203125f, 8.375f, -21.9375f, 17.625f, -9.2734375f, 0.923828125f, 4.11328125f, -4.26953125f, 20.953125f, -18.640625f, -1.24609375f, -18.046875f, 2.318359375f, 15.9375f, -4.02734375f, -10.796875f, 10.5703125f, 5.28515625f, -5.82421875f, 8.8515625f, 8.3984375f, 1.0029296875f, 14.3203125f, 6.4375f, -8.953125f, 24.296875f, -7.83203125f, -7.4140625f, 5.73046875f, 9.328125f, 17.234375f, -14.8515625f, 5.33984375f, -20.9375f, -8.640625f, -11.984375f, -12.140625f, 6.015625f, -17.40625f, 20.5625f, 3.837890625f, 0.365234375f, -12.3984375f, 3.744140625f, -3.974609375f, 0.306884765625f, 0.92236328125f, 1.900390625f, 4.0234375f, -2.451171875f, -12.296875f, -1.458984375f, 6.40625f, -6.46484375f, -14.84375f, 4.53125f, -2.484375f, 0.08184814453125f, 20.375f, -1.7109375f, -10.3515625f, 7.59375f, -11.2578125f, 1.0322265625f, -25.671875f, -15.7734375f, -5.81640625f, -8.28125f, -1.3427734375f, -36.09375f, -9.1015625f, 15.71875f, 22.59375f, -13.3515625f, 4.7109375f, -8.015625f, 8.53125f, 25.0f, -18.703125f, -20.0625f, 7.97265625f, 10.5078125f, -13.8359375f, -0.98095703125f, 1.5078125f, -1.9775390625f, -16.578125f, 8.2421875f, 5.91796875f, 10.96875f, -3.208984375f, 25.796875f, -13.9921875f, 0.51220703125f, 1.48046875f, 28.109375f, 10.53125f, 1.396484375f, -5.26171875f, 0.1014404296875f, -18.140625f, -10.1171875f, -4.62890625f, 15.3984375f, 0.8623046875f, 10.375f, 12.5625f, 21.5f, 8.625f, 0.258056640625f, 15.4609375f, 6.453125f, -1.9873046875f, -0.32421875f, -0.9619140625f, 15.3359375f, -12.8984375f, 5.74609375f, -7.69921875f, -15.0f, 4.09375f, 15.9609375f, 1.5927734375f, -1.6591796875f, 13.3984375f, 1.4453125f, 18.46875f, 6.76953125f, 20.859375f, 0.7900390625f, 3.70703125f, 5.94140625f, -7.0390625f, -17.90625f, 3.640625f, 11.390625f, 13.9453125f, 11.3515625f, -0.8359375f, 2.7421875f, 10.8046875f, 13.09375f, -1.8974609375f, 11.8125f, -2.87890625f, 22.28125f, 1.8857421875f, -12.4765625f, -3.478515625f, -13.34375f, -6.15234375f, -6.2890625f, -2.953125f, 0.33544921875f, 16.5f, -12.4453125f, -15.2265625f, 9.03125f, -6.15234375f, 7.6796875f, -0.97705078125f, -8.1796875f, -9.5f, 30.59375f, -3.154296875f, 9.96875f, 14.5078125f, 7.515625f, -26.859375f, -13.03125f, 5.86328125f, 6.4296875f, -5.1484375f, 13.75f, 9.796875f, 11.59375f, -7.63671875f, 20.0625f, 4.36328125f, -19.515625f, -7.140625f, -5.84765625f, -6.9140625f, 8.7265625f, 9.8515625f, 0.70654296875f, -15.328125f, -10.40625f, 0.07659912109375f, 11.765625f, -1.7236328125f, -4.19921875f, 19.40625f, 20.171875f, 4.12890625f, -2.267578125f, -15.4921875f, 6.80078125f, -4.62109375f, -18.90625f, -3.703125f, -24.8125f, -4.88671875f, -15.640625f, 6.0f, -23.5625f, 1.48046875f, -8.1328125f, 13.3203125f, -21.09375f, 0.55908203125f, -11.5546875f, 4.63671875f, 2.4140625f, 11.4765625f, -24.796875f, -3.763671875f, -1.423828125f, 3.322265625f, 9.1796875f, 19.625f, 3.86328125f, 8.0546875f, 2.7109375f, 4.2265625f, -0.41650390625f, -12.390625f, 17.421875f, -19.671875f, -12.9375f, -13.6015625f, -12.359375f, 0.9443359375f, 2.740234375f, -14.7421875f, 2.978515625f, -10.03125f, -8.53125f, 4.24609375f, -25.5625f, -15.1171875f, 22.8125f, 18.828125f, -32.28125f, 7.0078125f, 3.09765625f, -19.953125f, 9.8046875f, 6.328125f, 7.703125f, 6.60546875f, 8.6328125f, 5.0078125f, 0.7138671875f, 1.2265625f, 19.453125f, 10.4609375f, 8.7578125f, 5.79296875f, -2.01171875f, 1.4208984375f, 5.23828125f, 2.91796875f, 19.296875f, 0.32470703125f, 2.84765625f, 7.05859375f, 3.470703125f, -6.23046875f, -3.634765625f, -20.46875f, 11.671875f, 19.078125f, -6.65234375f, 8.0f, 1.375f, 4.77734375f, -18.609375f, 2.5078125f, 12.03125f, 11.171875f, -17.1875f, 41.1875f, 7.3671875f, -3.572265625f, 7.54296875f, 3.62109375f, 1.9052734375f, 10.359375f, 0.73681640625f, -12.375f, 0.1168212890625f, 4.9921875f, -12.1484375f, -4.34765625f, 25.828125f, -3.59765625f, 4.8515625f, -13.1015625f, -19.671875f, 6.24609375f, -12.6015625f, 3.853515625f, 25.859375f, -8.46875f, -9.8046875f, 6.76171875f, 2.087890625f, -8.3984375f, 15.125f, -0.271240234375f, 1.7958984375f, -6.06640625f, -1.7744140625f, -6.03515625f, 0.42138671875f, -18.234375f, -16.453125f, -1.521484375f, -2.455078125f, 18.828125f, -7.88671875f, 7.38671875f, 0.7939453125f, 1.494140625f, 2.2421875f, 2.587890625f, 10.65625f, -9.671875f, 4.5859375f, 9.3125f, 3.998046875f, 7.3046875f, -6.52734375f, 15.6015625f, -3.07421875f, -0.359619140625f, -1.6279296875f, 8.4765625f, 6.625f, -11.0234375f, 4.25390625f, -4.04296875f, -0.29345703125f, -12.984375f, 3.705078125f, 4.44921875f, -19.6875f, 11.2734375f, 10.140625f, 16.390625f, 5.01953125f, 12.5390625f, -25.09375f, -0.64599609375f, 16.609375f, -6.40234375f, -7.203125f, 6.8984375f, -12.71875f, -4.78515625f, -15.015625f, 10.9296875f, -15.9375f, 3.849609375f, 4.1484375f, -0.18408203125f, -6.94921875f, 3.189453125f, 6.6796875f, 2.44921875f, -0.28515625f, 19.96875f, -13.9375f, -8.84375f, 2.57421875f, -4.9609375f, 4.125f, 1.177734375f, 15.46875f, -1.4541015625f, -4.88671875f, -19.578125f, -6.140625f, -5.00390625f, -20.28125f, -12.171875f, -18.796875f, -4.9921875f, -2.337890625f, 17.921875f, 11.359375f, 17.296875f, 11.3984375f, -5.3515625f, -7.5625f, -5.68359375f, 7.71875f, 3.9609375f, -11.2421875f, 1.4501953125f, 3.939453125f, -6.04296875f, -0.0026950836181640625f, -14.15625f, -16.75f, -4.765625f, 7.9765625f, 12.6953125f, 14.2421875f, 3.345703125f, -2.4765625f, -15.7109375f, -0.9462890625f, -5.47265625f, -5.546875f, -5.09765625f, -26.0f, 2.830078125f, -0.77001953125f, 3.1484375f, -5.2578125f, -5.5703125f, -3.802734375f, 3.705078125f, 30.359375f, 3.9765625f, -12.34375f, -9.2578125f, 11.5f, -10.4375f, 13.9375f, -2.69140625f, 7.171875f, -13.203125f, 7.67578125f, -9.625f, -0.7685546875f, 8.5625f, 12.4609375f, -4.66015625f, -20.1875f, -3.494140625f, 3.005859375f, 19.671875f, 1.9013671875f, 1.5791015625f, 24.28125f, -9.4765625f, 11.9375f, 9.8515625f, -3.201171875f, 3.43359375f, 5.70703125f, 9.7265625f, -5.10546875f, 3.625f, 10.3515625f, 2.857421875f, -5.30078125f, -4.92578125f, -0.3427734375f, -1.8388671875f, -1.4970703125f, -3.130859375f, 16.5625f, 6.92578125f, -6.125f, -6.421875f, 8.296875f, 7.16796875f, -16.953125f, 2.087890625f, -26.8125f, 29.25f, -15.1328125f, -1.837890625f, 27.8125f, -5.98828125f, 15.0546875f, -2.3515625f, -2.123046875f, 17.453125f, 7.76953125f, -11.625f, 9.9296875f, -4.3125f, 17.546875f, -2.53515625f, 7.8828125f, -3.091796875f, 0.375f, -16.546875f, -11.34375f, 1.1513671875f, 11.1328125f, 1.0302734375f, -4.44921875f, -21.375f, -0.069580078125f, 0.93798828125f, 16.5625f, -12.515625f, -9.828125f, -13.7265625f, 20.953125f, 3.3984375f, 25.859375f, -16.515625f, 14.3671875f, -34.0f, -3.296875f, 18.5f, 12.9140625f, -2.3203125f, -2.97265625f, 11.890625f, -21.71875f, 15.6015625f, -5.2734375f, -1.14453125f, -22.6875f, 15.8125f, 0.8046875f, 2.2265625f, 7.859375f, -8.59375f, 1.30078125f, 8.1640625f, 1.6455078125f, 3.341796875f, 1.4638671875f, 5.0234375f, -18.375f, 23.578125f, -1.5498046875f, 16.609375f, 2.626953125f, 2.99609375f, 6.171875f, 4.4921875f, -10.8828125f, 10.3828125f, 0.71142578125f, -8.40625f, -5.96875f, 0.329833984375f, 8.578125f, 6.07421875f, 25.140625f, -4.90625f, -0.99609375f, -2.248046875f, 25.15625f, -24.703125f, 3.220703125f, 3.9921875f, 25.953125f, -9.3671875f, -1.630859375f, 7.8125f, -7.77734375f, 11.9375f, -14.96875f, 8.6796875f, -6.78515625f, 1.2880859375f, -29.890625f, -0.8623046875f, -2.591796875f, 1.8583984375f, 10.671875f, 4.00390625f, 15.890625f, -3.197265625f, 3.857421875f, 2.4921875f, -4.51171875f, 5.2109375f, 11.953125f, 17.40625f, 0.0445556640625f, 21.0f, -6.08203125f, -10.765625f, 6.6640625f, 1.3642578125f, -2.01953125f, 0.03094482421875f, -1.1767578125f, -1.8095703125f, -13.3984375f, -10.7734375f, 17.03125f, -11.46875f, -3.859375f, 7.421875f, 8.2265625f, -4.13671875f, -21.9375f, 31.875f, 13.96875f, 17.15625f, -17.03125f, -4.0625f, 1.912109375f, 11.3359375f, -0.65576171875f, 3.5546875f, -0.43896484375f, -0.17822265625f, 6.078125f, -5.4765625f, 2.337890625f, 4.85546875f, 0.66064453125f, -5.6328125f, -11.8203125f, -5.7734375f, -1.3984375f, -2.76953125f, -6.62890625f, -9.2734375f, 5.81640625f, -20.28125f, 3.73046875f, -0.33642578125f, -12.046875f, 12.921875f, -11.5703125f, 0.061248779296875f, 13.53125f, 6.81640625f, -19.765625f, -8.2265625f, 13.640625f, -7.40625f, -14.0078125f, -7.640625f, 2.25390625f, -2.12890625f, 5.64453125f, -19.78125f, -11.9609375f, 8.75f, -10.15625f, 8.4375f, 4.26953125f, 7.0390625f, 12.109375f, -4.734375f, -0.30224609375f, -14.2109375f, 22.5625f, -2.478515625f, -8.4140625f, 4.7109375f, -6.7421875f, 8.625f, 7.6015625f, -1.4267578125f, -5.8515625f, 18.171875f, 2.626953125f, -4.84765625f, 32.9375f, 8.9375f, 9.734375f, 9.890625f, -10.4453125f, 0.6298828125f, -6.2734375f, 6.23828125f, -3.09375f, -10.59375f, -1.47265625f, 17.78125f, 0.884765625f, -3.37890625f, 16.046875f, -8.953125f, 27.4375f, -12.65625f, 11.8828125f, 6.30078125f, 18.75f, 20.109375f, 1.462890625f, 2.62109375f, -2.998046875f, 8.1640625f, -33.71875f, -14.96875f, -3.560546875f, 3.396484375f, 15.5390625f, -0.11346435546875f, -1.0419921875f, 17.96875f, -5.34375f, -8.234375f, 3.078125f, 2.8671875f, -0.121826171875f, 7.7109375f, -11.65625f, -21.390625f, -2.619140625f, -18.0f, 1.615234375f, 1.9072265625f, -0.354736328125f, 10.0703125f, -10.8046875f, 14.5078125f, -9.8125f, 18.265625f, -3.70703125f, 4.23828125f, 10.0234375f, -3.498046875f, -19.375f, 5.9609375f, 27.875f, 20.875f, -2.44140625f, -17.6875f, 8.7890625f, -4.27734375f, -0.61767578125f, -9.671875f, -1.048828125f, 2.09375f, 13.609375f, -14.609375f, -1.9990234375f, -5.94921875f, -9.109375f, -15.6171875f, -6.078125f, 5.9375f, -4.26171875f, 11.8359375f, 15.1640625f, -11.9765625f, -4.5078125f, -9.359375f, -0.1339111328125f, -6.0f, 0.403564453125f, 2.546875f, -2.390625f, 12.984375f, -2.271484375f, -7.859375f, 8.1171875f, -11.4375f, -10.5234375f, 3.583984375f, 5.81640625f, -9.765625f, 16.078125f, -1.8583984375f, -17.046875f, -2.4609375f, 6.078125f, 15.1953125f, 11.890625f, 3.21484375f, -4.9453125f, 1.2412109375f, -6.375f, 0.1317138671875f, 3.259765625f, 11.4375f, -1.4423828125f, 12.46875f, -10.015625f, -23.25f, -8.3984375f, -3.224609375f, -10.421875f, -24.4375f, -0.5419921875f, -3.46875f, 6.453125f, -1.0634765625f, -5.0859375f, -12.953125f, -16.875f, 11.828125f, -12.4765625f, -10.3046875f, -21.546875f, -6.375f, -4.35546875f, -1.8427734375f, -14.7109375f, -17.921875f, -2.607421875f, -2.88671875f, 12.0390625f, 13.75f, 11.0703125f, -5.9921875f, 2.958984375f, -4.57421875f, -17.21875f, 12.1484375f, 5.4375f, 0.6171875f, 0.76318359375f, 10.8515625f, 3.048828125f, 20.6875f, -17.53125f, 0.70263671875f, 9.484375f, -18.734375f, -16.765625f, 17.59375f, -5.06640625f, 18.171875f, 12.3359375f, -0.403564453125f, 10.3671875f, -4.6875f, -12.46875f, -16.46875f, -10.3828125f, -2.95703125f, -11.671875f, 21.984375f, 20.234375f, -0.027130126953125f, 4.36328125f, -6.7265625f, 20.140625f, 10.703125f, 8.34375f, -9.6484375f, -26.671875f, 12.46875f, -17.515625f, 2.857421875f, 1.6982421875f, 3.17578125f, -21.453125f, -4.44921875f, -9.4453125f, -3.345703125f, -13.4296875f, 27.75f, 4.70703125f, -2.46484375f, -13.859375f, 9.9140625f, 16.0625f, 10.953125f, 19.953125f, -1.3115234375f, 1.2568359375f, 13.859375f, -20.21875f, -0.040283203125f, 0.931640625f, 0.302001953125f, 16.171875f, -11.9609375f, -0.46826171875f, -8.4453125f, 3.46875f, -6.4140625f, -8.484375f, -16.234375f, 8.28125f, -14.796875f, -11.9296875f, -14.859375f, -0.1556396484375f, -29.71875f, -5.70703125f, 2.634765625f, -5.703125f, 7.859375f, -5.8515625f, 7.1171875f, -13.984375f, -8.390625f, 2.9140625f, -4.3203125f, -23.078125f, -20.84375f, -2.822265625f, 3.619140625f, 4.11328125f, 14.4921875f, -18.125f, 12.125f, -6.78515625f, -5.2890625f, -9.1015625f, 12.609375f, 9.5625f, 1.1142578125f, 2.0078125f, -25.671875f, -4.03125f, 9.6875f, 6.3828125f, -5.45703125f, 7.40625f, 7.80859375f, 12.640625f, -29.78125f, -16.71875f, -0.10015869140625f, 3.830078125f, -11.03125f, -8.1640625f, 20.625f, -7.09375f, 17.75f, -0.90283203125f, -18.328125f, -7.6484375f, -38.34375f, -14.5234375f, 1.6845703125f, 14.328125f, -11.1484375f, -8.6171875f, -10.328125f, -7.19921875f, -4.74609375f, 18.65625f, 0.059173583984375f, 0.7001953125f, -7.18359375f, -0.3359375f, -5.85546875f, -13.546875f, 12.546875f, -2.171875f, -18.515625f, -19.1875f, -9.078125f, 32.21875f, -2.28125f, 8.265625f, 4.53125f, 10.7890625f, -1.4228515625f, -3.876953125f, -14.296875f, 7.203125f, -13.78125f, 8.984375f, 6.1875f, 7.44140625f, -4.01171875f, -9.734375f, -2.705078125f, -8.6328125f, -6.23828125f, 6.57421875f, 2.505859375f, -7.37109375f, 5.84765625f, -12.75f, 3.3359375f, -11.9375f, -14.2109375f, -8.5625f, 1.2314453125f, 0.7607421875f, 0.947265625f, 11.0078125f, 6.81640625f, -21.703125f, 0.90966796875f, 18.0625f, 26.5f, -0.9990234375f, -10.6015625f, 11.3203125f, 15.3984375f, 3.15625f, 12.5859375f, 4.92578125f, -8.671875f, 11.2421875f, 5.69140625f, -2.69921875f, -4.66796875f, 0.96337890625f, -16.375f, 3.107421875f, 5.203125f, -2.810546875f, 3.38671875f, 2.681640625f, 7.05078125f, 33.46875f, -7.078125f, -1.6826171875f, -2.87890625f, -0.81201171875f, 0.07806396484375f, -7.6171875f, -5.70703125f, 1.9873046875f, 18.0625f, -8.9453125f, 10.84375f, -2.94140625f, 4.52734375f, 0.1162109375f, -3.09765625f, -11.1953125f, 20.125f, 5.6796875f, -20.25f, -6.2890625f, -1.2548828125f, -3.98046875f, 8.203125f, 10.5234375f, 1.9765625f, 1.7353515625f, 1.298828125f, -8.4140625f, 0.9599609375f, 5.6875f, -6.73046875f, -2.36328125f, -13.359375f, -2.8046875f, 6.41796875f, 6.61328125f, -12.3203125f, -2.681640625f, -8.9921875f, 26.421875f, -13.2421875f, -17.53125f, 9.03125f, -2.765625f, 49.40625f, -2.9453125f, 3.189453125f, -4.52734375f, 7.015625f, -21.5f, 3.39453125f, 17.0f, 1.1650390625f, -8.859375f, -7.76953125f, -38.28125f, 4.6796875f, -7.87890625f, -1.96484375f, 11.875f, -22.296875f, -15.84375f, -2.19140625f, 0.68212890625f, -20.671875f, 21.765625f, -15.6796875f, 2.90625f, -11.0390625f, 0.352294921875f, 9.2421875f, -7.078125f, -9.90625f, 3.98828125f, -5.1875f, 7.05078125f, 3.921875f, -20.953125f, 20.625f, -5.59375f, -7.05078125f, -2.560546875f, -3.591796875f, -19.34375f, 0.947265625f, 9.296875f, 1.8046875f, -5.36328125f, -11.8515625f, 1.2626953125f, 11.2109375f, -14.5546875f, 4.06640625f, 13.84375f, -4.15234375f, -7.5078125f, -9.5f, -4.234375f, -7.6796875f, -5.02734375f, 6.25390625f, 9.0546875f, 15.78125f, 6.85546875f, -13.265625f, 11.8984375f, -6.1328125f, 10.3984375f, -5.98828125f, -4.0234375f, 2.37890625f, -11.7578125f, 13.15625f, -13.3125f, 15.28125f, 0.4482421875f, -9.9453125f, -2.548828125f, -2.01171875f, -23.65625f, -14.4765625f, 7.94140625f, -3.271484375f, 5.71875f, 20.5f, -26.734375f, -22.125f, -7.17578125f, 6.7890625f, -6.90234375f, 5.4375f, 24.25f, 4.98828125f, 3.0234375f, 2.333984375f, -2.828125f, 20.515625f, 3.9921875f, -26.21875f, 19.546875f, -10.5f, 10.78125f, 27.171875f, -10.5f, -3.310546875f, -23.484375f, -8.8359375f, -8.3203125f, 5.703125f, -4.48828125f, 3.919921875f, -3.40625f, -16.578125f, -17.59375f, -1.58203125f, 13.234375f, -2.271484375f, -1.896484375f, 22.46875f, 2.55078125f, -6.578125f, -3.328125f, -9.359375f, 10.3203125f, 6.203125f, -4.125f, -1.55078125f, 1.9326171875f, -7.42578125f, -15.0078125f, -1.9365234375f, 8.953125f, 4.1484375f, 16.046875f, 2.140625f, 18.125f, -3.025390625f, 15.6796875f, -12.28125f, 2.224609375f, 25.5625f, -14.375f, 6.875f, -20.53125f, 1.0947265625f, -4.37890625f, 9.40625f, 17.78125f, -15.5390625f, 7.9375f, 2.392578125f, -19.578125f, -8.9921875f, -7.16796875f, 8.4921875f, 7.6171875f, -5.5234375f, 6.46875f, 5.5703125f, 0.609375f, 10.6640625f, 6.74609375f, -15.25f, 8.671875f, -5.484375f, -9.6328125f, -3.18359375f, 2.767578125f, -4.34375f, 2.783203125f, -8.1640625f, -6.06640625f, -26.765625f, -0.6142578125f, -12.15625f, 1.2451171875f, 2.01171875f, -2.7734375f, -15.671875f, 11.0078125f, 5.0390625f, 1.501953125f, 3.134765625f, -14.7265625f, 1.4736328125f, 13.4609375f, -5.1328125f, -37.15625f, -2.732421875f, -19.453125f, -24.59375f, -15.8359375f, 8.1015625f, -5.9296875f, -16.90625f, -3.3828125f, -11.3046875f, 10.4921875f, -15.8828125f, 11.75f, -10.4296875f, -3.568359375f, -16.90625f, 8.4140625f, 1.6630859375f, -18.9375f, -12.3125f, 0.314208984375f, -7.390625f, -23.5625f, 17.4375f, 3.0625f, 2.2890625f, -0.9287109375f, 24.125f, 0.1339111328125f, -11.9453125f, -23.40625f, -7.77734375f, -27.96875f, 1.314453125f, 7.734375f, -0.81396484375f, 1.7353515625f, 24.375f, 1.6494140625f, -31.96875f, -4.94921875f, -7.765625f, -19.234375f, -4.83203125f, -1.4736328125f, 3.158203125f, -9.328125f, -27.96875f, -0.74609375f, -2.720703125f, -0.46875f, 5.37890625f, -4.5234375f, -20.703125f, -4.69921875f, 3.564453125f, 18.109375f, -3.35546875f, -11.0234375f, 23.015625f, 8.4140625f, -20.1875f, 14.1328125f, 13.4609375f, 14.234375f, 12.7734375f, -3.8046875f, 6.6171875f, 13.515625f, -16.703125f, -17.453125f, -0.62109375f, -2.94140625f, -2.798828125f, -2.478515625f, 7.5f, 0.97265625f, 7.125f, -7.36328125f, -3.154296875f, -0.11785888671875f, 4.9921875f, -2.63671875f, 19.28125f, -9.2890625f, 17.546875f, 2.146484375f, -8.7578125f, 9.453125f, -6.671875f, -1.923828125f, -21.375f, 4.0234375f, 16.453125f, 18.59375f, -0.712890625f, 23.953125f, 4.89453125f, 15.109375f, 13.4375f, -0.380859375f, -14.0703125f, -9.0234375f, 4.703125f, -27.21875f, 4.43359375f, -8.8515625f, 9.1171875f, -2.32421875f, -1.2900390625f, -5.89453125f, 20.34375f, 5.2109375f, -1.15625f, 3.421875f, -0.499755859375f, -21.875f, -0.9375f, -0.94140625f, 10.6484375f, -2.88671875f, 2.4921875f, -10.875f, 11.765625f, 2.337890625f, 7.24609375f, 1.9716796875f, -13.9296875f, -5.90625f, 5.30859375f, 13.1484375f, 4.56640625f, -1.4755859375f, -16.890625f, -6.3046875f, 21.640625f, -3.35546875f, -25.875f, 10.5703125f, 6.1796875f, -2.21484375f, 1.1259765625f, -1.29296875f, 4.08984375f, 2.548828125f, -12.859375f, 10.0859375f, 5.4921875f, 10.1875f, 5.9765625f, 0.3115234375f, -5.47265625f, -1.140625f, 1.8896484375f, -3.23046875f, 3.654296875f, 2.736328125f, 14.359375f, 3.892578125f, 21.5625f, -5.640625f, 9.65625f, 7.26171875f, -3.078125f, -15.5703125f, -6.546875f, -8.0234375f, 3.58203125f, -10.1875f, -19.890625f, 10.0390625f, -9.2890625f, -8.078125f, -7.87109375f, -2.87890625f, 14.1875f, -12.6484375f, -28.859375f, 14.296875f, -0.31787109375f, 11.6953125f, 7.57421875f, 0.45166015625f, -17.359375f, 0.5615234375f, 1.8095703125f, 4.390625f, -8.125f, 19.046875f, -15.3125f, -4.57421875f, 3.37109375f, 6.796875f, -2.173828125f, 4.81640625f, -12.9140625f, 19.203125f, -18.21875f, 8.0234375f, 1.5009765625f, 16.75f, 3.556640625f, 9.765625f, -5.51171875f, -13.484375f, 5.55859375f, 8.328125f, 5.71484375f, -14.1953125f, -2.623046875f, -8.7890625f, -0.025299072265625f, -9.9375f, 0.2344970703125f, 2.4921875f, 8.40625f, 3.28515625f, 2.62890625f, 6.27734375f, 18.1875f, 7.0f, 12.3828125f, -3.703125f, 3.41796875f, 3.232421875f, -4.71875f, -1.9384765625f, -1.8095703125f, 4.0f, 13.609375f, 8.171875f, -23.15625f, 18.59375f, -2.05078125f, -7.05078125f, 18.65625f, 3.53515625f, -9.0859375f, -17.984375f, -2.861328125f, 19.234375f, -7.234375f, 5.734375f, -7.14453125f, -0.36865234375f, 16.703125f, -1.6357421875f, -13.296875f, -11.46875f, 9.0390625f, 4.66796875f, 15.7265625f, 3.677734375f, 14.8125f, 7.30078125f, 3.712890625f, -10.0f, 1.8896484375f, -10.359375f, 3.2890625f, 18.140625f, -30.640625f, -28.0625f, -0.1292724609375f, -7.98828125f, 2.775390625f, 11.4609375f, 8.53125f, -11.671875f, -3.091796875f, -5.72265625f, 0.0247039794921875f, 1.541015625f, -5.5546875f, 10.6796875f, -9.65625f, 18.0f, 3.408203125f, 1.880859375f, -0.89111328125f, -6.1015625f, 8.609375f, 1.1552734375f, 9.921875f, -15.859375f, -0.38427734375f, -7.26171875f, -1.7119140625f, -1.3623046875f, 10.1796875f, -12.0078125f, -9.375f, -6.79296875f, 9.6328125f, 4.3046875f, 7.046875f, 13.7734375f, 8.84375f, -8.0078125f, 10.484375f, -23.0625f, -1.8740234375f, 15.609375f, 12.0234375f, -9.1484375f, 0.35595703125f, -0.51806640625f, -5.4765625f, -14.2265625f, -1.8857421875f, -12.6171875f, 6.859375f, 14.6015625f, 7.86328125f, 8.2109375f, -0.41943359375f, 10.546875f, -28.484375f, 1.94140625f, 13.53125f, 4.66015625f, -1.458984375f, -7.734375f, -6.4609375f, 15.40625f, -5.01171875f, -2.19921875f, -12.828125f, -1.8779296875f, -12.34375f, 6.94140625f, -14.9609375f, 6.234375f, 3.705078125f, -0.032012939453125f, 7.87109375f, 3.65234375f, 11.78125f, -1.294921875f, 24.140625f, -10.5f, 13.2421875f, -0.50634765625f, -4.9140625f, 11.25f, 10.1171875f, -3.255859375f, -17.34375f, 12.578125f, 10.390625f, 4.6640625f, -5.55859375f, 1.125f, 7.2890625f, -15.6015625f, 11.234375f, -5.66015625f, 0.489501953125f, -11.453125f, -6.078125f, -5.28125f, 1.6064453125f, -5.38671875f, 6.07421875f, -1.7392578125f, 5.38671875f, -17.328125f, 7.40234375f, 7.96484375f, -13.0234375f, 12.0234375f, -1.94140625f, -1.8154296875f, -10.3515625f, 8.234375f, -7.51953125f, -20.484375f, -3.0625f, 5.98046875f, 12.2578125f, 20.578125f, -0.93603515625f, -1.7919921875f, 7.44921875f, -19.890625f, 9.5f, 1.2900390625f, -6.7421875f, 0.09515380859375f, 4.125f, 21.796875f, -7.54296875f, 9.765625f, -7.2890625f, 5.828125f, 2.6875f, -8.0703125f, 16.96875f, -4.19921875f, -8.171875f, -0.65283203125f, 13.0f, 10.4375f, -15.5078125f, -14.0703125f, 12.4296875f, 0.93994140625f, -6.53125f, 5.42578125f, -0.402099609375f, -13.390625f, -2.73046875f, 9.4296875f, -1.927734375f, -20.40625f, 1.58984375f, 22.609375f, 11.28125f, -6.8125f, -7.0546875f, -1.6904296875f, -19.5f, -5.1328125f, -1.7197265625f, 16.140625f, -5.703125f, 3.96484375f, 9.625f, 9.015625f, 10.109375f, -20.6875f, 28.03125f, -22.96875f, 1.345703125f, 7.45703125f, -9.984375f, -4.78515625f, 17.734375f, 25.8125f, -2.017578125f, 5.71875f, -2.548828125f, -5.8984375f, 41.1875f, 13.0703125f, 15.5859375f, -2.5546875f, -11.359375f, 5.64453125f, 14.125f, -4.69921875f, 1.60546875f, 3.416015625f, -2.77734375f, -7.89453125f, 8.390625f, -2.1953125f, 2.412109375f, 30.859375f, 1.7275390625f, -0.82177734375f, -6.921875f, 4.921875f, 2.123046875f, -8.8828125f, 6.38671875f, 19.296875f, 0.95751953125f, -10.1484375f, -1.716796875f, 2.982421875f, -11.1328125f, 3.162109375f, -3.5f, -13.2734375f, -10.90625f, 4.89453125f, 6.78125f, -10.7734375f, -2.693359375f, 2.849609375f, -11.34375f, -18.953125f, 4.21875f, 19.046875f, 4.71484375f, 7.33984375f, 12.78125f, -8.1328125f, -6.109375f, 4.90625f, 1.041015625f, 1.3212890625f, -10.7890625f, 0.8095703125f, -3.623046875f, -0.77880859375f, -6.4375f, 5.89453125f, 12.6875f, -23.484375f, -6.625f, 8.09375f, -10.5078125f, 5.15234375f, 12.8125f, 1.8134765625f, 11.0078125f, -0.458740234375f, -0.70361328125f, 2.365234375f, 4.0390625f, 10.1484375f, 5.77734375f, 10.1015625f, 7.859375f, -3.388671875f, -15.9921875f, 32.6875f, -4.9453125f, 4.8671875f, 3.732421875f, -1.6708984375f, 0.73388671875f, -9.75f, 4.53515625f, -1.5849609375f, 2.884765625f, -2.44921875f, 5.328125f, 34.625f, -3.8359375f, 0.0418701171875f, -17.484375f, 4.7578125f, -12.8046875f, -11.84375f, 11.2578125f, -7.0703125f, -1.9482421875f, 2.427734375f, 5.79296875f, -1.92578125f, 5.203125f, 14.2578125f, -13.2734375f, 16.40625f, 5.765625f, -2.59375f, -12.2421875f, -3.271484375f, -28.4375f, -0.339111328125f, 1.7626953125f, -2.974609375f, -1.07421875f, 3.572265625f, 9.4921875f, -5.5625f, 2.689453125f, -3.26171875f, -2.90234375f, -3.134765625f, 3.703125f, -15.3515625f, 4.42578125f, 1.59375f, 2.404296875f, 6.70703125f, 7.2734375f, -17.859375f, 20.890625f, -0.275146484375f, -10.2265625f, 22.234375f, 12.421875f, 1.5341796875f, 11.8359375f, -1.2744140625f, -11.8125f, -6.65234375f, -1.6591796875f, -30.484375f, -5.25f, -15.5f, 11.921875f, 9.9375f, 2.240234375f, -2.107421875f, 8.6328125f, -2.833984375f, -13.875f, -14.1640625f, -19.4375f, 13.3125f, -8.375f, -16.203125f, 11.21875f, 7.99609375f, -20.234375f, 0.93408203125f, 2.830078125f, -1.8125f, -13.453125f, 8.046875f, -17.5f, 7.07421875f, -1.955078125f, 5.63671875f, -9.7890625f, -10.0859375f, -4.953125f, 24.0f, 18.953125f, 10.9375f, 7.41796875f, -20.296875f, -2.76171875f, 12.9609375f, -14.0625f, 6.25390625f, -20.21875f, -4.25390625f, -12.6328125f, 19.5f, 34.9375f, 9.6328125f, 20.015625f, 3.16015625f, 5.390625f, 3.10546875f, 8.8671875f, 10.140625f, -10.5625f, 12.625f, 6.87109375f, -5.7109375f, -1.0244140625f, 11.984375f, 4.84765625f, -7.328125f, -15.7890625f, -4.81640625f, -24.25f, -6.57421875f, -5.06640625f, 4.953125f, -3.365234375f, -14.4921875f, -5.953125f, 29.953125f, -11.3359375f, 6.92578125f, -14.1171875f, -8.640625f, 14.1875f, 11.109375f, -1.3818359375f, -16.984375f, 4.80859375f, -0.97802734375f, -25.859375f, 13.59375f, 0.247802734375f, 16.5625f, -1.0576171875f, 14.6484375f, -8.25f, -11.0703125f, -8.6640625f, -6.3203125f, -3.244140625f, 7.3203125f, -12.0703125f, 3.349609375f, 1.44921875f, 0.42578125f, -7.63671875f, -4.9921875f, 1.7275390625f, 0.07470703125f, 21.796875f, -1.953125f, -0.474853515625f, -13.296875f, -2.013671875f, 1.5029296875f, 3.033203125f, -23.578125f, 6.08203125f, -9.0390625f, -3.234375f, -24.40625f, -3.212890625f, -10.7265625f, -6.51171875f, -20.640625f, -6.11328125f, -7.37109375f, 6.13671875f, 7.03515625f, -15.5390625f, 2.65625f, -11.421875f, 3.994140625f, -8.609375f, 3.328125f, 13.515625f, -1.4091796875f, 7.54296875f, -8.6484375f, 7.30859375f, -4.58984375f, -2.8984375f, 0.93212890625f, -30.859375f, -2.916015625f, -6.83984375f, -3.46484375f, 12.9765625f, -4.921875f, 1.0078125f, 7.3671875f, 4.8671875f, -8.0234375f, -5.51953125f, 10.1171875f, 16.046875f, 5.20703125f, 8.0390625f, -5.13671875f, -17.546875f, -5.99609375f, 9.3125f, 3.5f, -1.8994140625f, 11.140625f, -5.765625f, -0.83447265625f, -16.390625f, -18.453125f, -6.8984375f, 7.73046875f, 4.04296875f, 3.41015625f, -12.734375f, 20.078125f, -4.59375f, 3.029296875f, -26.140625f, 0.501953125f, -14.6484375f, 11.203125f, -3.583984375f, 25.765625f, -22.15625f, 11.9453125f, 19.734375f, -24.21875f, 6.0390625f, 8.296875f, -3.201171875f, -0.6953125f, 7.37109375f, 3.228515625f, 1.875f, 30.734375f, -3.41796875f, 11.015625f, -7.5546875f, 11.9140625f, -7.1875f, 4.859375f, 2.447265625f, -11.0078125f, 27.40625f, -8.8828125f, 10.8359375f, 17.640625f, 14.328125f, 0.015167236328125f, -2.455078125f, -7.25390625f, -2.435546875f, 8.1640625f, -11.7734375f, -13.3125f, 3.818359375f, 12.953125f, -4.9375f, 6.0703125f, 10.7578125f, 6.16015625f, -23.578125f, 15.0390625f, 14.7734375f, 5.94140625f, 8.921875f, 10.21875f, -4.53515625f, -14.015625f, -22.328125f, -6.85546875f, 7.4921875f, 4.22265625f, -24.953125f, -5.73828125f, 7.73046875f, -16.90625f, 4.03515625f, -24.734375f, -1.37109375f, 6.92578125f, 21.515625f, 7.27734375f, -23.96875f, 2.095703125f, 5.41796875f, 15.9453125f, 6.08984375f, -3.41015625f, 1.814453125f, -5.62109375f, -13.828125f, 0.1710205078125f, 12.25f, 12.1328125f, -5.54296875f, 2.203125f, -28.328125f, -3.68359375f, 3.712890625f, 0.95068359375f, -7.37890625f, -0.359619140625f, 17.984375f, -1.0029296875f, -7.4140625f, 8.9453125f, 4.28515625f, -2.056640625f, 2.548828125f, -11.1640625f, 12.53125f, 5.984375f, 12.78125f, -9.890625f, 4.8515625f, -1.4541015625f, -2.169921875f, 3.802734375f, -1.0712890625f, -15.78125f, 5.17578125f, -8.8515625f, -6.51171875f, 14.9140625f, 13.265625f, -11.640625f, 1.248046875f, -12.8828125f, -7.5859375f, -8.734375f, 18.078125f, 6.82421875f, 5.5f, 11.015625f, 1.158203125f, 19.859375f, 1.7578125f, 0.129150390625f, -4.85546875f, 22.421875f, -10.515625f, 4.234375f, -9.3203125f, -1.3759765625f, 2.365234375f, -5.640625f, -5.421875f, -3.70703125f, -3.568359375f, -11.5546875f, -5.56640625f, -21.984375f, -9.078125f, -2.36328125f, 6.2578125f, 3.640625f, 31.234375f, 5.40234375f, -11.5859375f, -14.65625f, -7.76953125f, 3.880859375f, 14.1640625f, 11.7421875f, 6.2109375f, -16.578125f, 3.099609375f, -3.685546875f, -21.40625f, -13.5625f, 8.625f, -0.58740234375f, -0.90673828125f, 2.443359375f, 0.2841796875f, 7.1640625f, 3.267578125f, -3.767578125f, -13.2421875f, -20.25f, -0.316650390625f, 31.21875f, 7.0703125f, 12.65625f, 12.515625f, -3.046875f, -31.609375f, -14.1796875f, 13.6875f, 13.3828125f, 5.078125f, 0.418212890625f, -6.703125f, -7.078125f, 7.046875f, -7.34375f, -2.90625f, 10.2109375f, -7.38671875f, 8.109375f, 9.609375f, 3.330078125f, -0.224853515625f, 6.05078125f, -11.109375f, 12.5703125f, 13.3046875f, 9.8203125f, -13.609375f, -1.3740234375f, -1.5263671875f, 13.265625f, -11.4375f, 7.20703125f, -11.8984375f, 3.296875f, -20.84375f, 0.90380859375f, -5.1171875f, 4.52734375f, 6.58203125f, 8.0234375f, -3.115234375f, -15.5390625f, -11.78125f, 12.359375f, -7.625f, -8.984375f, -10.296875f, 3.158203125f, -2.95703125f, -13.140625f, 21.328125f, 4.890625f, 9.9375f, -9.578125f, -15.1875f, 11.46875f, 21.828125f, 12.7421875f, -9.2421875f, 1.9912109375f, -31.171875f, 7.15625f, -1.2421875f, -16.578125f, 11.3203125f, 0.31591796875f, -9.4921875f, -4.61328125f, -5.890625f, -10.9609375f, 7.203125f, 11.0390625f, 7.625f, -19.6875f, -9.3671875f, -6.12890625f, -24.9375f, 3.453125f, -2.369140625f, -9.8671875f, 10.4140625f, -1.7470703125f, -2.16796875f, 1.4033203125f, -0.95654296875f, 2.39453125f, -15.4375f, -10.4609375f, -3.5234375f, 4.38671875f, 1.681640625f, -10.1328125f, -7.109375f, 21.328125f, -4.95703125f, -15.3359375f, -2.865234375f, 3.05078125f, -20.484375f, 14.859375f, -20.265625f, -10.4375f, -31.28125f, 4.9375f, -5.07421875f, 8.828125f, 0.64501953125f, 5.77734375f, -1.8037109375f, 0.058807373046875f, 3.240234375f, 10.234375f, 4.2109375f, 13.3125f, 4.48046875f, 4.19140625f, 8.7421875f, 2.21875f, 7.375f, -12.265625f, -2.736328125f, -2.421875f, 15.8984375f, -4.62890625f, -19.203125f, 7.40234375f, 12.15625f, -0.85693359375f, 17.0f, 1.6083984375f, 22.46875f, -1.134765625f, 1.970703125f, 10.0546875f, -21.5625f, 18.0625f, -5.21484375f, -24.171875f, 0.9306640625f, -12.078125f, -7.81640625f, -7.40234375f, 3.1953125f, -4.14453125f, -3.513671875f, 6.4453125f, -0.23095703125f, -5.05078125f, 11.2890625f, -2.330078125f, 10.7109375f, -0.07159423828125f, -3.0859375f, 0.035369873046875f, 5.6875f, -24.8125f, -16.65625f, -8.75f, 1.6083984375f, 0.79443359375f, 16.390625f, 5.125f, 10.671875f, -7.578125f, 5.56640625f, -8.5390625f, -24.78125f, 22.625f, 3.0703125f, -2.005859375f, -2.958984375f, -16.734375f, 2.755859375f, 7.99609375f, -21.59375f, 10.8515625f, 7.6484375f, -0.217529296875f, 2.33203125f, 9.7578125f, 9.8984375f, -2.142578125f, -5.921875f, 8.328125f, 14.0f, 13.0390625f, -7.4375f, -14.3125f, 4.859375f, -19.171875f, -15.3359375f, 10.5703125f, -9.3046875f, -16.0f, 9.3828125f, 10.1171875f, 10.390625f, -0.8974609375f, 14.203125f, 1.6201171875f, -0.426025390625f, 11.578125f, 19.203125f, 12.2109375f, -2.044921875f, -8.203125f, 6.5390625f, -5.1015625f, 23.125f, -3.900390625f, -19.5625f, -5.046875f, 4.70703125f, -32.84375f, -1.6669921875f, 26.96875f, 22.578125f, -1.0771484375f, 6.11328125f, 3.103515625f, -22.046875f, 8.5390625f, 1.3603515625f, -8.5703125f, 0.67431640625f, 9.1484375f, 5.3046875f, -0.6552734375f, 5.1640625f, 12.9296875f, 4.921875f, 12.0390625f, -1.046875f, -11.6171875f, 8.5f, -0.79833984375f, -12.5390625f, 10.03125f, -17.0f, 22.046875f, -6.44140625f, -6.265625f, 12.1875f, -7.6953125f, -1.0927734375f, 12.078125f, -1.453125f, -2.65625f, -12.8984375f, -1.955078125f, -4.20703125f, -6.90234375f, -12.640625f, 4.78515625f, 0.77685546875f, -4.421875f, 29.6875f, -14.5390625f, 5.1015625f, -14.453125f, 6.57421875f, 12.4609375f, -11.75f, 15.1796875f, -4.1328125f, 6.23046875f, 7.3359375f, -1.697265625f, 3.73828125f, 9.8515625f, 3.64453125f, 2.123046875f, -3.541015625f, -1.1630859375f, 5.4609375f, 3.40234375f, 0.7138671875f, 2.345703125f, -23.671875f, 5.078125f, 10.6953125f, 0.0809326171875f, -1.1787109375f, 12.8671875f, 2.876953125f, -12.78125f, -21.25f, -19.1875f, 18.625f, -8.796875f, -12.515625f, 5.28125f, -0.423828125f, 0.169677734375f, 4.1953125f, -8.609375f, -1.162109375f, 6.2890625f, -0.498291015625f, 9.3359375f, 12.59375f, -9.609375f, 0.935546875f, 10.96875f, 2.37109375f, 27.328125f, -12.1328125f, -2.060546875f, -3.89453125f, 18.46875f, 12.2265625f, -6.8515625f, -11.84375f, 1.69921875f, -4.125f, -3.818359375f, -12.7265625f, -7.80078125f, 14.84375f, -10.453125f, 4.99609375f, -7.328125f, 15.375f, -3.556640625f, -7.328125f, 7.51171875f, -7.28515625f, 29.046875f, 3.7421875f, 8.546875f, 16.890625f, 20.46875f, -4.484375f, -27.9375f, 27.453125f, -1.064453125f, 7.453125f, 5.9453125f, -11.765625f, -5.41015625f, 8.6484375f, -6.8515625f, 3.623046875f, -1.732421875f, 1.1162109375f, -3.69921875f, -0.05584716796875f, -8.3515625f, 1.111328125f, 12.5703125f, -6.6796875f, 2.994140625f, -14.59375f, 9.328125f, 9.5625f, 8.75f, 9.1953125f, 1.6875f, -3.22265625f, 26.359375f, -2.06640625f, -13.7734375f, -0.7998046875f, 15.7890625f, 18.9375f, -15.515625f, -27.09375f, 6.1875f, -2.525390625f, -2.30078125f, 1.888671875f, -1.4248046875f, 5.25f, 5.453125f, -8.2734375f, 15.4296875f, 7.00390625f, -6.4609375f, -8.046875f, -12.203125f, -8.109375f, 1.5986328125f, 2.86328125f, -0.36669921875f, 2.681640625f, -8.2578125f, -1.796875f, 2.26953125f, 7.59375f, -1.1865234375f, -20.4375f, -13.5390625f, -24.109375f, 0.87646484375f, -7.3515625f, -12.78125f, -4.015625f, -16.015625f, 9.875f, 6.390625f, 10.09375f, 11.6640625f, -8.609375f, -6.09375f, -14.59375f, 10.5703125f, 8.921875f, 14.7734375f, -12.796875f, -4.51171875f, 0.488037109375f, -10.421875f, 5.51171875f, 3.392578125f, 12.359375f, -3.169921875f, 9.3125f, -12.8828125f, -8.7734375f, 6.25f, 2.4296875f, 19.4375f, -6.23046875f, -7.25390625f, 4.7109375f, 8.4921875f, -19.109375f, 10.4140625f, 8.171875f, -6.8984375f, 2.345703125f, -5.53125f, 21.96875f, 3.609375f, -10.890625f, -19.5625f, 3.728515625f, -4.98046875f, -15.9375f, 2.947265625f, 11.203125f, 8.84375f, -4.875f, 6.84375f, -3.40625f, 7.91015625f, 18.65625f, 8.1328125f, 8.53125f, -7.21875f, 29.484375f, 5.46484375f, 11.5f, 19.328125f, -9.953125f, 4.7890625f, 13.234375f, -10.6796875f, -8.890625f, 6.609375f, 7.078125f, -6.23828125f, -1.04296875f, -1.046875f, -12.578125f, -3.796875f, 19.5f, 0.369140625f, 3.173828125f, -6.0f, -7.8828125f, 8.7109375f, -14.0f, 3.728515625f, -5.8203125f, -7.484375f, -2.373046875f, -19.890625f, 18.65625f, -0.478271484375f, -10.03125f, 2.3984375f, -10.625f, 19.359375f, -2.46484375f, -4.9296875f, -6.3125f, 12.390625f, -2.7265625f, 3.349609375f, 0.296875f, 13.75f, -9.53125f, -10.3515625f, 3.416015625f, -0.46240234375f, -14.0703125f, -6.21875f, 14.8515625f, -13.3828125f, 4.29296875f, -2.65625f, 3.84765625f, 5.24609375f, -7.6015625f, 8.3828125f, -2.23828125f, -18.375f, -11.1640625f, -19.71875f, -8.1484375f, -11.0546875f, 4.67578125f, 3.939453125f, 5.2265625f, -9.28125f, 6.30859375f, 2.6015625f, -11.03125f, 6.88671875f, -11.703125f, 12.5546875f, 10.9609375f, -13.140625f, -5.8359375f, 4.5078125f, 4.39453125f, -22.65625f, 15.65625f, 9.7109375f, 4.609375f, -13.546875f, 2.431640625f, -2.6484375f, 0.912109375f, -5.10546875f, 13.3515625f, 2.146484375f, -22.390625f, 0.1668701171875f, 4.2265625f, -8.1875f, 15.7265625f, 13.296875f, 3.3828125f, -8.828125f, 17.578125f, 4.8359375f, -5.83203125f, 8.8359375f, 5.33203125f, 22.484375f, -10.265625f, 15.6953125f, 20.328125f, 1.6513671875f, -13.2421875f, -21.0f, -2.37890625f, 9.9140625f, -5.92578125f, -1.0673828125f, 3.744140625f, 14.0390625f, 10.140625f, 2.138671875f, -3.509765625f, 22.28125f, 7.4609375f, 0.4365234375f, 8.1171875f, -4.515625f, -3.880859375f, 2.630859375f, 25.875f, 1.3134765625f, -9.9375f, 12.09375f, -7.796875f, 2.455078125f, -2.494140625f, -6.12890625f, 3.693359375f, 11.2265625f, -0.01800537109375f, 9.0078125f, -12.5234375f, -10.96875f, 1.90234375f, -3.814453125f, 17.828125f, 6.1640625f, -4.75390625f, 31.921875f, 3.1640625f, -7.12890625f, -10.84375f, -7.49609375f, -2.806640625f, -11.2578125f, 9.9140625f, 3.208984375f, 13.0390625f, 14.421875f, 7.29296875f, 4.140625f, 1.4970703125f, -15.9765625f, -13.6328125f, 5.58984375f, -12.765625f, -19.765625f, -15.3203125f, 10.453125f, -4.03515625f, 9.890625f, 1.357421875f, -1.0703125f, -8.1171875f, 29.359375f, -0.87646484375f, -8.03125f, 2.568359375f, -11.234375f, -13.4921875f, -10.9375f, 8.1484375f, -10.84375f, -11.8515625f, -10.9140625f, -1.8564453125f, 12.046875f, -1.64453125f, -6.80859375f, -2.044921875f, -6.25f, -8.59375f, 16.84375f, 7.953125f, -8.203125f, -4.42578125f, 11.078125f, -16.203125f, 12.2734375f, -3.65625f, -12.125f, -12.7734375f, -4.828125f, 14.6875f, 5.87890625f, -12.4375f, -11.4375f, -1.4658203125f, 17.03125f, 8.0625f, -1.6279296875f, -9.515625f, 1.306640625f, 5.46875f, -6.5859375f, 4.6328125f, 6.8671875f, 9.203125f, 2.029296875f, 1.3662109375f, 2.38671875f, 15.0859375f, -12.03125f, -8.125f, -2.404296875f, -4.36328125f, -1.892578125f, 6.15234375f, -1.201171875f, 18.515625f, 10.625f, -3.423828125f, 7.0234375f, 5.9921875f, 7.95703125f, 4.37109375f, 2.95703125f, 2.791015625f, 6.25390625f, 1.7744140625f, -9.828125f, -15.0703125f, -2.78515625f, 0.95751953125f, 13.484375f, 2.927734375f, -3.0703125f, 7.046875f, -0.99609375f, 3.255859375f, -17.375f, -2.361328125f, 8.7109375f, -2.826171875f, 4.28125f, 1.4052734375f, -0.65380859375f, 9.4765625f, -3.19921875f, 16.265625f, -14.1796875f, 7.890625f, -8.484375f, -1.5439453125f, -6.49609375f, 0.1314697265625f, 0.97900390625f, 0.1572265625f, -1.6552734375f, 2.736328125f, 12.0625f, 2.9765625f, -2.01953125f, 4.421875f, -17.6875f, 0.365234375f, 9.1328125f, -14.1796875f, 24.015625f, 13.453125f, -9.59375f, 16.46875f, 8.2578125f, -22.609375f, -14.390625f, -11.1171875f, 1.794921875f, 0.294677734375f, 5.30078125f, 14.796875f, -10.8671875f, 9.625f, -1.724609375f, -7.56640625f, 29.8125f, 2.763671875f, -18.921875f, 16.90625f, -11.78125f, -3.24609375f, -1.4345703125f, 19.96875f, -12.6953125f, 12.40625f, -4.00390625f, -4.046875f, 7.41015625f, -5.48828125f, -5.1796875f, 4.6796875f, -4.48828125f, -7.375f, -6.125f, -11.6015625f, -5.56640625f, 0.61474609375f, -26.0625f, 20.515625f, 15.046875f, -6.22265625f, 16.875f, -3.8671875f, 8.03125f, 14.515625f, 8.890625f, -18.203125f, -1.59765625f, -19.234375f, 20.8125f, -3.119140625f, -21.4375f, 10.265625f, -7.3984375f, -4.21875f, -25.671875f, -2.96875f, 13.328125f, -6.5859375f, 15.3125f, -7.05078125f, 5.05078125f, 9.5546875f, -4.03125f, -22.8125f, -1.556640625f, -10.640625f, -0.9140625f, -19.421875f, 19.75f, 0.381103515625f, -18.15625f, 8.171875f, 9.1015625f, 10.9296875f, 4.19921875f, 2.177734375f, -18.703125f, -19.078125f, -6.2421875f, -18.015625f, -1.3671875f, -14.5234375f, -2.58984375f, 1.4404296875f, -9.2578125f, -10.8046875f, -4.99609375f, -8.625f, -0.81396484375f, 18.8125f, -4.8984375f, 11.2109375f, -13.6015625f, -6.375f, -11.0078125f, 12.3046875f, 8.7265625f, -13.109375f, 0.73828125f, -2.044921875f, 9.125f, -12.96875f, 1.15234375f, -5.8828125f, -15.6640625f, -13.1796875f, 13.8515625f, 0.94189453125f, -5.5390625f, 9.515625f, -0.197265625f, 4.3828125f, -10.3125f, -17.046875f, 9.8515625f, 21.171875f, -13.2578125f, 4.5859375f, 16.453125f, 14.2578125f, 10.3203125f, 2.9453125f, -24.71875f, -9.015625f, -14.828125f, 3.6484375f, -2.12109375f, 8.140625f, -4.12890625f, -3.517578125f, -9.8515625f, -3.71484375f, -10.9921875f, 4.21875f, 1.236328125f, 7.16015625f, 0.3125f, 33.625f, -2.5234375f, 4.11328125f, 7.1484375f, -25.84375f, -26.1875f, 14.6171875f, -2.416015625f, -11.2578125f, 10.828125f, 7.79296875f, -3.203125f, -4.57421875f, -3.408203125f, 0.41943359375f, 9.8046875f, 9.8046875f, -27.4375f, 11.234375f, -0.220703125f, 6.58984375f, 8.8125f, -12.6953125f, 4.671875f, -2.041015625f, 1.2646484375f, -17.609375f, 3.90625f, 1.0849609375f, 6.98046875f, 1.732421875f, -4.94140625f, -1.078125f, -6.5546875f, -0.092529296875f, 3.38671875f, 9.4140625f, 25.890625f, 13.765625f, -19.6875f, 11.0234375f, 2.681640625f, -10.5859375f, 2.6484375f, 10.421875f, -11.578125f, -6.96484375f, 20.625f, 0.292236328125f, 5.96875f, 10.3984375f, 9.3203125f, -3.966796875f, 14.6484375f, 1.9111328125f, 1.34375f, 5.8515625f, 6.13671875f, 1.595703125f, 3.951171875f, -6.390625f, 8.9609375f, 10.9609375f, -5.62890625f, 19.65625f, 2.494140625f, 1.6611328125f, -1.666015625f, 11.5234375f, 8.234375f, 6.18359375f, -0.1402587890625f, -10.6640625f, 18.90625f, -16.265625f, -3.841796875f, 15.5f, -10.7734375f, -1.6240234375f, 4.5234375f, 28.8125f, 21.4375f, 18.59375f, -15.9921875f, 14.59375f, 15.7578125f, 1.4814453125f, -6.07421875f, -14.453125f, -1.638671875f, 6.83984375f, -6.67578125f, 5.48046875f, -5.28515625f, -8.328125f, 19.828125f, 0.693359375f, 0.68701171875f, -3.83203125f, 5.2265625f, -7.5546875f, 13.3203125f, 7.39453125f, 11.609375f, 2.04296875f, -12.6640625f, 0.2398681640625f, 1.8828125f, 5.22265625f, -20.0625f, 5.2421875f, 1.109375f, -10.265625f, -2.853515625f, 6.46875f, 3.001953125f, 7.80859375f, 14.2109375f, -2.01171875f, 3.169921875f, 2.744140625f, -3.6953125f, -2.45703125f, 5.53515625f, -14.9140625f, -6.3515625f, -2.642578125f, 0.1800537109375f, 14.921875f, 8.453125f, 15.984375f, -8.765625f, 11.34375f, 9.5546875f, -5.140625f, 1.87109375f, -10.5078125f, -0.437255859375f, 5.76171875f, -5.48828125f, -5.8515625f, -0.0933837890625f, 0.6171875f, -16.671875f, 10.2890625f, -21.421875f, -1.78125f, 10.7421875f, -4.6484375f, -2.59375f, 4.21875f, -11.484375f, -4.62890625f, -13.4296875f, -8.203125f, -4.62890625f, 12.1484375f, -6.35546875f, 19.171875f, 14.0078125f, 27.515625f, 16.234375f, -0.86572265625f, -7.84765625f, 2.806640625f, 3.263671875f, -9.5078125f, -13.90625f, 9.1953125f, 7.62109375f, -31.21875f, 2.7265625f, -1.9306640625f, -2.94140625f, -0.03204345703125f, 2.36328125f, 1.7685546875f, -12.0703125f, 10.59375f, 2.4140625f, -10.8515625f, 2.0859375f, -10.0703125f, 10.3046875f, 12.9140625f, 2.5703125f, 4.62890625f, -2.671875f, 0.055633544921875f, 7.26953125f, -27.3125f, -8.1484375f, 11.7734375f, 14.3046875f, -1.7109375f, -7.4921875f, 13.53125f, 4.453125f, 8.5703125f, 2.830078125f, -1.3544921875f, 1.9033203125f, 23.578125f, -2.197265625f, 0.9677734375f, -10.4296875f, -5.09765625f, -8.921875f, 9.8671875f, -34.03125f, 11.2734375f, -0.61181640625f, 14.953125f, -4.8671875f, -9.875f, -6.32421875f, -8.515625f, -3.390625f, -1.541015625f, 16.3125f, 9.59375f, 3.685546875f, 17.359375f, 10.9453125f, 12.7109375f, -0.00557708740234375f, 10.125f, 2.248046875f, -2.224609375f, -6.6796875f, -4.1015625f, -7.671875f, 14.9140625f, 6.70703125f, -2.701171875f, -8.7734375f, 3.48046875f, -13.171875f, -3.6328125f, -3.328125f, -4.109375f, -11.5546875f, 1.01171875f, 16.890625f, 3.125f, 0.841796875f, 7.875f, -9.2734375f, -6.98046875f, 7.0546875f, -2.244140625f, 12.359375f, -0.36669921875f, -2.572265625f, -8.34375f, 7.9921875f, -6.890625f, -13.1484375f, 11.296875f, -4.8984375f, 3.759765625f, 18.8125f, -18.8125f, -5.03125f, -10.859375f, -6.578125f, -5.0390625f, -20.046875f, 14.609375f, 18.125f, 8.671875f, 4.55078125f, -15.421875f, -9.6015625f, 3.08203125f, 8.359375f, -6.484375f, -11.0546875f, 3.759765625f, -0.53076171875f, 18.78125f, -19.328125f, -0.435546875f, -5.3046875f, -0.3505859375f, -8.9765625f, 6.28515625f, 6.78125f, -16.125f, 5.44921875f, -3.01171875f, -12.3125f, -11.78125f, 7.453125f, -6.74609375f, -1.6669921875f, 4.53125f, -11.953125f, 7.890625f, -26.703125f, -5.4921875f, 4.421875f, -3.37890625f, -4.49609375f, 10.578125f, 14.4453125f, 6.1875f, -4.78125f, -8.359375f, 2.845703125f, 7.3515625f, 6.953125f, 18.09375f, -2.578125f, 26.28125f, -7.3671875f, 15.484375f, 19.3125f, 24.453125f, -3.5625f, -3.5234375f, 1.2666015625f, -29.546875f, 10.953125f, -8.3515625f, -8.203125f, 1.59375f, 8.359375f, -1.3544921875f, -4.75390625f, -2.853515625f, -9.4921875f, 6.62890625f, 7.96875f, -2.064453125f, 3.794921875f, 3.291015625f, 5.2421875f, -4.93359375f, -3.265625f, 26.421875f, 3.57421875f, -32.375f, 2.216796875f, 11.5703125f, -9.40625f, -14.3125f, -5.55859375f, 5.16796875f, 12.8515625f, 16.609375f, 15.6875f, 5.60546875f, -8.6953125f, -12.5234375f, 10.8046875f, -4.08984375f, -2.81640625f, 3.92578125f, -9.9921875f, 12.3515625f, -12.0703125f, 15.2890625f, -3.6171875f, -18.390625f, 2.365234375f, -3.583984375f, 7.2734375f, -7.05078125f, -9.2890625f, -5.6953125f, -2.83984375f, 3.328125f, -13.90625f, -0.360595703125f, 11.9140625f, -10.1796875f, -0.042327880859375f, 15.9375f, -4.16015625f, 5.1015625f, -1.65625f, -1.220703125f, 7.45703125f, 17.109375f, 3.4609375f, -8.0f, 0.0660400390625f, 18.859375f, -8.4921875f, -24.03125f, -9.421875f, -3.09765625f, -3.25f, -2.701171875f, -12.8515625f, -5.6796875f, 4.83203125f, -2.537109375f, -5.6953125f, -23.984375f, -3.591796875f, -26.78125f, 3.560546875f, 16.71875f, -9.5546875f, 3.814453125f, 6.3125f, -9.765625f, -10.6015625f, 5.65625f, 11.4375f, -9.984375f, -0.411376953125f, 12.28125f, 0.646484375f, 21.015625f, -32.375f, -4.52734375f, -19.0f, -0.87255859375f, -1.5771484375f, 13.8671875f, -1.68359375f, -6.94921875f, 12.65625f, -15.90625f, 5.01171875f, -11.65625f, 5.05078125f, -13.203125f, 0.400146484375f, 19.171875f, 19.484375f, -1.875f, -13.3359375f, -0.29150390625f, -11.609375f, -6.4609375f, 5.59375f, -1.583984375f, -17.390625f, -10.7578125f, 7.16015625f, 12.1015625f, -4.671875f, 5.34375f, -8.515625f, -1.556640625f, 4.24609375f, 0.399169921875f, -2.880859375f, -7.29296875f, -7.55078125f, 0.8515625f, -1.1767578125f, 25.109375f, -2.63671875f, 5.515625f, 16.25f, -16.765625f, 2.919921875f, -7.5078125f, 6.83984375f, -2.1796875f, -10.0625f, -0.9931640625f, 6.4140625f, -19.421875f, 0.386962890625f, 7.390625f, -8.9609375f, -4.76171875f, -3.828125f, 10.0859375f, 18.515625f, 1.26171875f, 0.62158203125f, 1.1962890625f, -24.90625f, -9.96875f, 0.64013671875f, 4.76171875f, 3.03515625f, 13.3125f, 3.505859375f, -7.78125f, 2.91796875f, -4.10546875f, 4.921875f, -1.5302734375f, -9.921875f, 8.7578125f, 0.67431640625f, -2.630859375f, -9.6328125f, 3.390625f, 0.2415771484375f, -6.3203125f, -30.03125f, 6.453125f, -7.64453125f, 7.63671875f, -8.9609375f, 2.669921875f, 2.935546875f, -8.296875f, -15.28125f, -10.7265625f, -0.267578125f, 0.79052734375f, 13.34375f, -5.94921875f, -4.23828125f, -5.515625f, -3.0703125f, 4.40625f, 3.181640625f, 13.8125f, -2.19921875f, 1.1044921875f, -30.375f, -15.515625f, -6.20703125f, 15.4140625f, -26.96875f, -7.59375f, 0.2479248046875f, 13.0703125f, -6.40234375f, 8.859375f, 1.5751953125f, 0.06689453125f, 4.69140625f, 9.9375f, 3.6875f, -6.5234375f, 2.537109375f, -1.689453125f, 11.328125f, 6.34375f, 7.21484375f, -0.065185546875f, -7.26171875f, -15.0625f, 4.25f, -4.48828125f, 5.015625f, -4.34375f, -0.4248046875f, -17.9375f, -35.03125f, -0.52734375f, -7.59765625f, 6.98046875f, 8.15625f, -4.3046875f, -0.79443359375f, 13.828125f, -12.1171875f, 15.03125f, -15.171875f, -7.70703125f, -2.595703125f, 4.3984375f, -11.2421875f, -9.3984375f, 2.498046875f, 5.37890625f, 21.8125f, 7.80078125f, -3.287109375f, -29.796875f, 3.130859375f, -9.28125f, -16.046875f, 15.03125f, -8.40625f, 5.3828125f, 4.69140625f, 12.1484375f, -9.5546875f, 4.48828125f, -0.2386474609375f, -10.2578125f, -0.433349609375f, 1.7451171875f, 11.75f, -10.609375f, 6.296875f, 5.27734375f, -4.03125f, 1.2099609375f, -11.8359375f, 21.453125f, 3.630859375f, 5.05078125f, -0.1109619140625f, 9.3046875f, -0.6708984375f, 18.9375f, -22.65625f, 3.04296875f, -2.091796875f, 9.3046875f, -4.70703125f, 14.671875f, 5.625f, 9.5234375f, 16.4375f, 0.57958984375f, -6.14453125f, 16.65625f, 6.1640625f, -20.40625f, -16.578125f, 12.609375f, 14.6953125f, -7.01953125f, 2.982421875f, 26.078125f, -0.8046875f, -6.875f, -2.404296875f, 7.94140625f, -13.53125f, -8.7421875f, 3.447265625f, 15.6328125f, 3.927734375f, -4.02734375f, -3.03125f, 5.703125f, 3.48828125f, -8.0625f, -15.2890625f, -4.01171875f, -10.46875f, 7.01171875f, 4.79296875f, 8.5234375f, 1.21484375f, -3.380859375f, 3.810546875f, -7.91015625f, 1.6982421875f, 5.8046875f, 5.83203125f, -20.484375f, 4.7734375f, 0.0211029052734375f, 13.9765625f, 1.7841796875f, -0.50927734375f, 0.495361328125f, 6.125f, -4.484375f, -2.646484375f, 2.21875f, -5.39453125f, 17.78125f, 3.029296875f, -6.9140625f, -8.484375f, 7.4765625f, 7.33203125f, -10.2265625f, 11.7421875f, -2.95703125f, 10.734375f, 0.1688232421875f, -4.01171875f, -17.90625f, 8.8515625f, 0.1683349609375f, 15.0859375f, 1.8642578125f, 0.04345703125f, 0.61181640625f, -10.34375f, -18.625f, -3.814453125f, 11.125f, 3.33984375f, 9.40625f, 13.1640625f, -5.2890625f, 9.1328125f, -15.328125f}; - -static const __fp16 Apack_dram[8192] - __attribute__((section(".dram"), aligned(64))) = {0.1419677734375, 0.8125, -0.297119140625, 0.6748046875, -0.72119140625, -0.62646484375, 0.5966796875, 0.4951171875, 0.145751953125, 1.330078125, 1.0771484375, 1.421875, -1.9658203125, -1.9970703125, -0.2266845703125, -0.85205078125, -1.6689453125, 0.7734375, -1.521484375, 0.98193359375, -1.5, 1.0087890625, 0.027069091796875, -0.343994140625, -1.638671875, 0.64697265625, 0.45263671875, -0.74169921875, 0.062042236328125, -0.40625, -0.78564453125, -1.4453125, -1.33203125, -0.52294921875, -1.3916015625, -0.7998046875, 0.218994140625, 0.80517578125, 0.197509765625, 1.00390625, 0.9970703125, -0.40380859375, 2.404296875, 0.1181640625, 1.1689453125, 0.51171875, -1.37109375, 1.7001953125, 0.58251953125, 0.135986328125, -0.1173095703125, 0.97802734375, 1.9345703125, -1.318359375, 0.7607421875, -0.2568359375, -0.40869140625, 0.171630859375, -2.01953125, -0.69921875, 0.5068359375, -0.9560546875, 0.312744140625, 0.233642578125, 0.3740234375, -0.68896484375, -0.693359375, 0.110107421875, 1.4833984375, -0.16552734375, -1.9189453125, 0.5771484375, -1.28515625, 0.38525390625, -0.93408203125, 0.09088134765625, 0.22900390625, 1.4482421875, -1.8154296875, 1.0830078125, 1.755859375, 1.4609375, 0.541015625, 0.43212890625, 0.802734375, 0.9072265625, -2.00390625, 1.6171875, 0.1923828125, -0.26171875, 0.2998046875, -0.122802734375, 1.048828125, -1.8876953125, 0.92724609375, -2.701171875, 0.1396484375, 0.43798828125, -0.7939453125, -0.52001953125, -0.244140625, 0.198974609375, -1.4873046875, -0.45166015625, 1.48828125, -0.474853515625, -1.90234375, 0.46630859375, -0.415283203125, 1.056640625, -0.66259765625, -1.083984375, 0.419189453125, -0.8525390625, -0.6162109375, -0.36279296875, -0.2110595703125, -0.900390625, 0.271484375, 0.42822265625, 1.10546875, -0.658203125, 0.759765625, -1.0517578125, 0.86962890625, 0.69677734375, -1.095703125, 0.56396484375, -0.521484375, -0.045440673828125, 0.53466796875, -0.1632080078125, 0.348876953125, -0.06689453125, -2.3203125, -0.98974609375, 1.2333984375, 0.00745391845703125, -0.79443359375, 0.07086181640625, -0.5712890625, -1.5732421875, -0.66259765625, -0.9931640625, -0.1815185546875, -0.325439453125, 1.3349609375, -2.28515625, -0.337646484375, -0.3388671875, 0.46728515625, -1.0810546875, -0.487060546875, 0.485595703125, -0.43798828125, -1.40625, 0.83935546875, 0.61572265625, 1.0693359375, 0.10198974609375, -0.8515625, -1.30078125, -0.359130859375, -0.431396484375, -0.1202392578125, 0.8369140625, -0.465087890625, 0.958984375, 0.43359375, 0.7451171875, -0.1650390625, 0.576171875, -0.10369873046875, 1.025390625, -0.65966796875, 1.7275390625, 2.568359375, 1.1767578125, -0.033782958984375, -0.265380859375, -0.67822265625, 0.80224609375, -0.79931640625, 0.6552734375, 0.71484375, -0.245361328125, -0.294921875, -0.412353515625, 1.23046875, -0.356689453125, -0.345947265625, 0.4912109375, 1.0556640625, -0.29345703125, -0.333251953125, -0.468994140625, -0.2318115234375, 1.138671875, 0.01279449462890625, -0.440185546875, -0.4345703125, 0.8466796875, -1.12890625, 0.806640625, 0.03961181640625, -0.8916015625, 0.1212158203125, -0.07769775390625, -0.732421875, -0.197265625, 2.85546875, -0.216064453125, -0.7607421875, -0.296875, -0.1907958984375, 1.5654296875, -0.349609375, 1.4228515625, -1.724609375, 1.7568359375, -1.7001953125, 0.11541748046875, -0.54345703125, -0.796875, -0.78662109375, -0.4833984375, -0.49462890625, 2.830078125, 0.04779052734375, 1.21875, 0.77783203125, 0.9404296875, 2.041015625, 0.8369140625, -1.92578125, -0.505859375, -0.3720703125, 1.048828125, -0.50146484375, 1.501953125, 1.40234375, -1.53125, 1.3046875, 0.0107574462890625, -0.42626953125, -1.064453125, 0.58056640625, -1.57421875, -2.1796875, -1.21484375, -0.366455078125, -0.028564453125, 0.21044921875, -2.0546875, -0.2374267578125, -0.50048828125, -0.34619140625, -1.6171875, 0.55908203125, 0.29296875, 0.1112060546875, 0.6279296875, 0.814453125, 0.322998046875, -0.95361328125, 1.421875, 0.580078125, -0.0222930908203125, 0.0054473876953125, 0.7333984375, 0.13720703125, -0.01548004150390625, 0.09136962890625, -0.04510498046875, -2.4140625, 0.5078125, 0.1138916015625, -0.2406005859375, 1.6591796875, 0.54248046875, 0.724609375, 1.1142578125, 0.1131591796875, -0.98388671875, -1.2060546875, 0.8017578125, 1.46875, -0.603515625, -1.1875, -0.271728515625, 0.082275390625, 0.463623046875, -0.32666015625, 1.4052734375, -0.8701171875, -0.90576171875, -0.374755859375, -0.72509765625, 0.4814453125, -0.1676025390625, -0.38232421875, -0.810546875, -0.376220703125, 0.7451171875, 0.409912109375, -1.6201171875, -0.056884765625, -0.29150390625, -1.0810546875, -1.66796875, -0.2015380859375, -0.96728515625, -0.334716796875, 1.337890625, 0.260009765625, -0.47607421875, -1.052734375, 2.650390625, -2.28515625, 0.8818359375, 0.455322265625, -0.1099853515625, 0.61669921875, 0.0274505615234375, 0.054412841796875, 0.6279296875, -0.9248046875, 1.037109375, 0.004024505615234375, -0.455322265625, -0.9072265625, -1.50390625, -0.7392578125, 1.193359375, -0.5126953125, 1.9228515625, 0.59716796875, 1.6943359375, -1.3046875, -1.8515625, 0.277587890625, 0.265869140625, -2.181640625, 0.436767578125, 0.7939453125, -2.71875, 2.3828125, 1.6845703125, 2.138671875, -0.5986328125, -0.7197265625, 0.247802734375, 0.3427734375, 0.3505859375, 2.11328125, -2.0390625, -0.8955078125, -1.6875, 1.7099609375, 2.24609375, -0.318115234375, 1.2744140625, 0.646484375, 0.1488037109375, -1.220703125, 0.070556640625, 0.9619140625, 0.07073974609375, 1.26171875, 0.374755859375, -0.52294921875, 0.1317138671875, 1.1728515625, 1.546875, -1.1904296875, -0.464111328125, 1.298828125, 1.0439453125, 1.1796875, -1.49609375, 0.57373046875, -1.2763671875, -0.341796875, 0.9775390625, 0.342041015625, -0.71728515625, 0.36376953125, -0.285400390625, 1.23046875, -0.488037109375, -0.273193359375, 0.28173828125, -1.2578125, 2.0078125, 1.1064453125, -1.5693359375, 1.375, -0.814453125, -1.3427734375, -1.056640625, 0.63037109375, 1.015625, -0.26513671875, 0.8955078125, 0.83740234375, 0.2156982421875, 0.51416015625, 0.11572265625, 1.005859375, -1.62109375, 0.3203125, -0.62353515625, 2.08984375, -0.56982421875, -0.6943359375, 0.41015625, 0.779296875, 0.83349609375, -1.466796875, -1.4111328125, -0.388427734375, -1.6767578125, 1.572265625, 0.92822265625, -0.640625, -1.19140625, -0.9921875, 0.2607421875, -0.35009765625, -1.2587890625, 1.3623046875, -0.281005859375, -0.97900390625, -0.11822509765625, -1.3369140625, 1.5771484375, 0.53564453125, 3.705078125, 0.03564453125, 0.1419677734375, -1.4970703125, -0.466552734375, 0.428466796875, 1.310546875, 0.263671875, -2.001953125, -0.389404296875, 0.260986328125, 0.4658203125, 1.11328125, 1.591796875, -1.439453125, -0.387451171875, 0.8203125, -0.1583251953125, 0.67578125, -0.2135009765625, 1.951171875, -0.80224609375, -0.8525390625, 1.6572265625, -0.51611328125, 0.14208984375, -1.4541015625, -0.94189453125, 0.295166015625, 0.34716796875, 1.3837890625, 1.4453125, -0.619140625, -0.41064453125, -0.423095703125, 0.54345703125, -0.71533203125, 0.11553955078125, -0.71044921875, 0.26611328125, -0.205322265625, -0.96240234375, 0.004810333251953125, -0.47314453125, -1.306640625, -1.1884765625, -0.6123046875, -0.91552734375, 1.087890625, -2.36328125, 0.229736328125, -1.6416015625, -1.431640625, 0.12255859375, -0.46337890625, -0.7802734375, 0.127197265625, -0.024871826171875, -0.79638671875, -0.1827392578125, 1.9814453125, 0.62255859375, -0.221923828125, 1.5712890625, -1.4140625, 1.1474609375, 0.8603515625, 1.6416015625, -0.52880859375, 0.91796875, 0.448486328125, 0.346923828125, 0.007293701171875, 1.939453125, -2.3359375, 0.07781982421875, -0.50390625, -1.369140625, -0.6669921875, 2.248046875, -1.6611328125, 0.0938720703125, -1.228515625, -0.5302734375, 2.4765625, 0.10589599609375, -0.89794921875, -0.113525390625, 0.427978515625, 0.104736328125, -0.29052734375, 1.1396484375, 0.362548828125, -0.67333984375, 1.8916015625, -0.70703125, 1.2431640625, -1.4228515625, 0.7666015625, -0.249755859375, 0.05804443359375, 0.5166015625, -0.1556396484375, 1.0517578125, 0.83447265625, -0.2156982421875, -0.353759765625, -1.66796875, 0.372802734375, -1.017578125, 0.03179931640625, -0.7431640625, 0.109130859375, 0.65478515625, 0.294677734375, 0.66552734375, 0.2061767578125, -1.6689453125, 0.0009708404541015625, -0.0628662109375, -2.03125, -0.705078125, -1.3125, -0.1558837890625, -1.80078125, 0.57666015625, -1.955078125, 0.48828125, -0.281494140625, -0.1566162109375, 0.3466796875, 1.1611328125, 1.1787109375, 0.7978515625, 0.2269287109375, -1.345703125, -0.8271484375, 1.9521484375, 0.053619384765625, 1.2138671875, -0.63330078125, -0.231689453125, -0.12274169921875, 0.0670166015625, -1.21875, 0.88525390625, -0.134521484375, 1.466796875, -1.1748046875, 0.07135009765625, 0.2479248046875, -1.3017578125, 0.03009033203125, 0.8720703125, 0.5517578125, 2.240234375, 0.335205078125, 0.03436279296875, 0.8154296875, -0.10614013671875, 1.1953125, -0.65087890625, -1.3349609375, -0.93798828125, -1.828125, -0.171630859375, -0.475830078125, 0.062347412109375, 0.480712890625, 0.2198486328125, -0.505859375, -0.25927734375, -0.66796875, 0.63525390625, 0.52197265625, 2.798828125, -0.04425048828125, 1.97265625, 1.03515625, 0.306884765625, -0.408203125, -0.6005859375, -1.2373046875, -0.362060546875, -0.95849609375, -0.310791015625, 0.8564453125, 0.399169921875, -0.0292205810546875, 0.599609375, 0.1314697265625, -0.53515625, 0.24609375, 0.07635498046875, 0.53271484375, 1.205078125, -1.2802734375, -0.3212890625, -0.19921875, 0.822265625, 2.048828125, -1.791015625, -0.1102294921875, -0.47216796875, -0.056243896484375, 0.0092010498046875, 0.1456298828125, 0.479736328125, 0.1748046875, -0.18701171875, -0.47314453125, -0.6669921875, -1.6005859375, -0.7998046875, -0.0015869140625, -0.533203125, -0.77490234375, -1.1552734375, -0.5615234375, -1.5576171875, 0.1243896484375, 2.037109375, 0.1455078125, -0.261962890625, -0.326416015625, 0.306640625, 0.64453125, -2.654296875, 0.97216796875, 0.264892578125, -0.358154296875, -0.169189453125, 1.87109375, -0.1419677734375, 0.1224365234375, -0.5703125, -1.1845703125, -0.7060546875, 0.6591796875, 1.7744140625, -0.29541015625, -0.56494140625, -1.8017578125, -0.42724609375, -0.89892578125, 0.1861572265625, -0.7734375, 1.7099609375, -1.4501953125, 0.2313232421875, -1.435546875, 0.71142578125, 1.6689453125, -0.15576171875, -1.05078125, -0.96044921875, -0.1732177734375, 0.351806640625, 0.466796875, 0.74658203125, 0.88525390625, -0.497314453125, -0.36474609375, -0.423095703125, -0.86376953125, -0.4296875, -0.77880859375, 0.10845947265625, 0.48828125, -0.72412109375, -0.216796875, 1.109375, 0.28759765625, 0.78515625, -2.78515625, 0.065185546875, -1.8369140625, -0.1693115234375, 1.2119140625, 0.005802154541015625, -2.087890625, -0.2479248046875, 0.77294921875, 1.3662109375, 0.277587890625, 0.323974609375, -0.423828125, 0.7333984375, 0.806640625, -1.21875, -2.30078125, 2.576171875, -0.1495361328125, -0.130126953125, -0.06878662109375, -0.5556640625, 0.52490234375, -0.90869140625, -0.767578125, -0.47509765625, -1.16015625, -0.60888671875, -0.7822265625, -0.546875, 0.00824737548828125, 1.1328125, -1.2529296875, -1.103515625, -0.8505859375, 0.33447265625, 0.62841796875, 0.0426025390625, 0.939453125, -1.0654296875, 0.251953125, -2.62109375, -0.2388916015625, 0.19873046875, -0.302001953125, -0.01409149169921875, 0.759765625, -0.1654052734375, -0.728515625, 0.02264404296875, 0.1253662109375, -0.5517578125, 0.6240234375, 1.052734375, 0.4140625, -0.2020263671875, -1.716796875, -1.2783203125, 1.3466796875, -0.85693359375, 0.1458740234375, 0.003749847412109375, -1.498046875, -0.0201263427734375, -1.4638671875, 0.619140625, -0.458251953125, 1.25390625, 0.72509765625, 1.5859375, 0.6044921875, 0.83740234375, -1.169921875, 0.480224609375, 0.6953125, -0.7265625, 0.308837890625, -0.2841796875, -0.7666015625, 0.970703125, -0.06353759765625, 0.99462890625, 1.7197265625, 0.04986572265625, -0.79052734375, 0.76318359375, 1.2724609375, 0.6435546875, -0.4599609375, -0.480224609375, 2.064453125, -0.0255889892578125, 0.9033203125, 1.2724609375, 1.1416015625, 0.60205078125, -0.252197265625, -0.59765625, 1.884765625, -0.4189453125, 0.99365234375, 2.287109375, -0.83642578125, 0.057037353515625, -0.53271484375, -0.3486328125, -0.0836181640625, 0.24853515625, 0.447021484375, 0.6669921875, 0.5029296875, -0.923828125, -1.0751953125, -1.6181640625, 1.8388671875, -0.293212890625, -0.65478515625, 0.290771484375, 0.7509765625, 0.8095703125, 1.71875, 0.263916015625, 0.38916015625, -2.033203125, 0.650390625, -1.2216796875, 0.470703125, -0.83544921875, 1.166015625, -0.06591796875, -1.6328125, 2.23828125, 0.96533203125, 0.0838623046875, 0.5966796875, -0.11981201171875, 1.154296875, -0.338623046875, -0.8720703125, 1.58203125, -2.05078125, -0.1512451171875, -1.4921875, -0.062103271484375, 0.564453125, -0.64453125, -0.349853515625, -0.444580078125, -0.615234375, 0.79833984375, -0.99609375, 0.50634765625, 0.9169921875, 0.51904296875, 0.5791015625, -1.34765625, -0.276123046875, 0.16650390625, 0.966796875, -0.01480865478515625, 0.2032470703125, -0.28515625, -0.57763671875, 1.4306640625, -0.52490234375, 1.984375, -0.8564453125, 1.2724609375, -0.44677734375, -0.047027587890625, 0.447509765625, 0.81982421875, 0.29638671875, 0.8017578125, -0.8642578125, -0.00833892822265625, 0.2587890625, -0.42626953125, -0.0078582763671875, -0.72900390625, 0.445556640625, 0.09356689453125, 0.0121307373046875, -0.63330078125, -1.044921875, 0.962890625, -0.33984375, 2.80859375, -1.6767578125, 0.06744384765625, 0.892578125, 0.313720703125, -0.1580810546875, -1.240234375, -0.368896484375, -0.401123046875, -0.681640625, 1.2861328125, -0.38037109375, 0.19140625, -1.1689453125, 0.1302490234375, -0.5263671875, 0.36572265625, 1.9267578125, -0.075927734375, 1.060546875, 0.025177001953125, 0.775390625, 1.19140625, -1.1640625, -0.88427734375, -1.4501953125, -0.290771484375, -0.431396484375, -1.013671875, 0.95361328125, 0.689453125, -0.2452392578125, -0.08770751953125, -0.56201171875, 0.486572265625, -0.6103515625, -1.9287109375, 0.2318115234375, -0.061370849609375, -0.149658203125, -2.228515625, -0.2413330078125, 0.47314453125, -1.0732421875, 0.82763671875, 0.04132080078125, -1.5185546875, -0.4345703125, -0.935546875, -0.01494598388671875, -1.994140625, 1.12109375, -1.0185546875, -0.7509765625, -1.1474609375, -0.98388671875, -0.8642578125, -1.630859375, -0.1837158203125, -0.333251953125, 0.405517578125, 1.0625, 2.0625, -1.2900390625, 1.5400390625, 0.327880859375, -1.5166015625, -0.182861328125, -3.0859375, -0.3896484375, -0.16015625, -1.2685546875, -1.076171875, -1.2421875, -0.6083984375, -0.79638671875, -1.0068359375, 0.4736328125, -1.21484375, -0.63232421875, -1.373046875, -0.830078125, 0.1666259765625, -1.3857421875, -1.5712890625, 0.73193359375, 0.36474609375, -0.54638671875, -1.4560546875, -1.708984375, -0.1484375, 0.107177734375, 0.9013671875, 0.9375, -0.2012939453125, -1.1455078125, -0.52880859375, 0.364501953125, 1.037109375, 0.521484375, 0.2354736328125, -0.84716796875, 0.4326171875, 0.61669921875, -1.388671875, -0.3876953125, 0.4287109375, 0.12432861328125, 1.1474609375, -1.0849609375, -1.9873046875, 0.440673828125, -0.7626953125, -0.85009765625, -0.7900390625, 0.136474609375, -0.8447265625, -1.1611328125, -0.1136474609375, 0.80419921875, 0.94873046875, 0.58984375, -1.33203125, 0.59521484375, -1.046875, 0.74609375, -0.346923828125, -1.98046875, 0.05975341796875, -1.076171875, 1.64453125, -1.3271484375, -1.5029296875, 0.495849609375, 0.28662109375, 1.166015625, -1.21484375, -0.4873046875, -0.117431640625, 0.41357421875, 0.994140625, 0.30517578125, 0.449951171875, -0.1544189453125, -0.2335205078125, -0.1842041015625, 1.86328125, -0.257568359375, -0.36279296875, 0.7900390625, 1.65234375, 0.28857421875, -0.55615234375, 0.325439453125, -0.7392578125, 0.6611328125, -0.61083984375, 0.2059326171875, -0.7646484375, -0.47998046875, -1.09375, 0.533203125, -0.318359375, 1.5166015625, 0.10040283203125, 0.720703125, -0.734375, -0.701171875, -0.1607666015625, 1.041015625, -0.591796875, -1.193359375, -0.65380859375, 0.1571044921875, 0.5712890625, 0.1827392578125, 1.203125, -0.27783203125, 2.42578125, -0.044677734375, 1.2294921875, -0.2607421875, -1.830078125, 1.3115234375, -0.49267578125, 0.427490234375, 0.1884765625, 0.2200927734375, -0.51513671875, 1.0732421875, 0.062164306640625, -0.486328125, -0.4228515625, 0.484375, -0.36328125, 0.68408203125, -1.3408203125, -0.279052734375, 0.73388671875, 0.426025390625, -0.748046875, -1.7626953125, -0.474853515625, 0.369140625, -0.32470703125, 1.1455078125, -0.53173828125, 2.349609375, -0.39892578125, -0.615234375, -1.134765625, 0.9306640625, -0.65283203125, 1.171875, -0.0162506103515625, -0.62353515625, -0.1839599609375, -1.416015625, 0.43505859375, 0.206298828125, 0.17431640625, -0.075927734375, 1.7412109375, 0.544921875, -1.6884765625, 0.0224609375, 0.64794921875, 1.7685546875, 1.15625, 0.92724609375, -0.07183837890625, -0.12237548828125, -1.501953125, -0.07159423828125, -1.189453125, 0.2744140625, -0.0609130859375, -2.32421875, -0.48681640625, 1.39453125, 0.82763671875, -1.4765625, -0.50390625, -1.0087890625, 0.1343994140625, -1.486328125, -0.4775390625, 1.9013671875, -0.36376953125, -0.80712890625, -2.529296875, -1.419921875, -3.12109375, 0.89013671875, -1.33984375, -0.75, -1.279296875, 0.405517578125, 0.2354736328125, -2.705078125, -1.03515625, -0.498291015625, -0.37744140625, 1.220703125, 1.576171875, -0.450927734375, -2.388671875, -0.1265869140625, 0.08526611328125, 0.685546875, -0.1170654296875, -1.3671875, -1.619140625, 1.0908203125, 1.22265625, -1.3994140625, 1.5751953125, 0.4697265625, -0.927734375, -0.09295654296875, 1.8330078125, 0.55859375, -0.1602783203125, -0.587890625, 1.1845703125, 1.7998046875, -0.7158203125, -1.3623046875, -0.60888671875, -0.33349609375, -0.7763671875, 0.458251953125, -0.671875, 0.41455078125, 1.3583984375, -0.31396484375, -0.6767578125, 2.033203125, -0.02374267578125, 0.0247039794921875, -0.00803375244140625, 1.5703125, 0.56298828125, -0.55126953125, 0.302978515625, 0.759765625, 0.59228515625, 0.92236328125, 1.546875, 1.083984375, 2.099609375, -0.403076171875, -0.759765625, 0.05474853515625, 0.0675048828125, 1.5986328125, -0.427001953125, -1.130859375, -0.333984375, 1.318359375, 2.9375, -0.48779296875, -1.0009765625, 1.1669921875, -0.69921875, 1.544921875, -0.8076171875, -0.826171875, -2.689453125, -0.2178955078125, 1.1044921875, -0.7529296875, -0.38623046875, 0.1795654296875, -0.20654296875, -1.357421875, 0.74658203125, 1.779296875, -0.197509765625, 0.66357421875, -0.09613037109375, 1.41015625, -0.51611328125, 0.9375, 1.2373046875, 0.8740234375, 0.1947021484375, -0.248291015625, -0.46630859375, 1.1591796875, 1.0244140625, -0.408203125, -0.39990234375, -0.85546875, -0.72119140625, 0.919921875, -0.5712890625, 2.10546875, -1.451171875, 0.362060546875, 0.233154296875, 0.04351806640625, -0.3837890625, -0.361328125, 0.57421875, -0.54541015625, 0.43310546875, 0.64208984375, -0.80859375, 0.327392578125, -0.56298828125, -1.5556640625, 0.12127685546875, -0.30810546875, -0.284912109375, -1.5947265625, -0.1983642578125, -0.56689453125, 0.876953125, -0.2100830078125, 1.4609375, 0.63232421875, 1.4052734375, 0.261962890625, 0.9931640625, -1.34765625, 0.43310546875, 1.91015625, 0.29443359375, 1.2958984375, 1.4033203125, 0.583984375, 0.7421875, -0.181640625, 0.623046875, -0.9853515625, -0.322021484375, -1.294921875, 0.401611328125, -1.1650390625, -0.7734375, -1.943359375, 0.472412109375, 1.2177734375, -0.2030029296875, 1.0810546875, -1.2255859375, 0.79833984375, 1.3232421875, 0.0163116455078125, 1.11328125, 1.640625, -0.78125, 1.306640625, 0.034576416015625, -1.23828125, -0.443359375, 1.873046875, 1.6533203125, 1.0439453125, -1.88671875, 0.56005859375, 1.517578125, 0.361083984375, -0.045196533203125, -0.8837890625, 0.260498046875, 1.015625, 0.55615234375, 0.357177734375, -1.224609375, 1.4140625, -1.3291015625, -0.02972412109375, -0.63232421875, -0.2310791015625, -0.6865234375, 1.6796875, 0.2685546875, -0.08856201171875, 0.26416015625, 0.1917724609375, -0.5908203125, 0.76953125, -0.78759765625, -0.1015625, 1.6669921875, -0.85107421875, -0.84326171875, 0.91650390625, -3.046875, -0.75146484375, 0.759765625, 0.2021484375, 0.093994140625, -0.09454345703125, -1.208984375, -0.0119781494140625, 0.88427734375, -1.060546875, -0.40234375, 0.9208984375, 0.1058349609375, -0.41064453125, 0.25439453125, 0.1500244140625, 1.1982421875, 0.318115234375, 0.72314453125, -1.490234375, -0.78369140625, -0.7001953125, -0.2459716796875, 1.4326171875, -0.46875, -0.650390625, 0.8642578125, -1.3271484375, 0.54345703125, -0.09765625, 0.419677734375, -0.330322265625, -2.30078125, -0.15234375, 1.259765625, -1.6611328125, 0.2261962890625, 1.31640625, 0.431884765625, 0.271484375, 0.59423828125, 0.64404296875, 0.50634765625, 1.3779296875, -0.7763671875, -0.33251953125, -0.67529296875, -0.384521484375, -0.69873046875, -0.463623046875, 1.3515625, 0.658203125, 1.662109375, 0.1029052734375, -0.66552734375, 2.6328125, 1.0693359375, -0.400146484375, -0.154052734375, -0.6396484375, 1.2109375, -0.6865234375, -0.040863037109375, -0.4375, 1.0400390625, -0.1829833984375, -0.08905029296875, -0.7705078125, 0.6572265625, -0.4365234375, 0.25439453125, 0.0343017578125, 1.0830078125, -2.693359375, 0.407470703125, 1.0634765625, 0.2305908203125, -1.1796875, -0.76171875, 1.08984375, -0.55859375, 0.292236328125, 1.0185546875, -1.6044921875, 0.8251953125, 0.1876220703125, 0.2958984375, -0.281005859375, 0.67724609375, 0.30712890625, -0.2158203125, -1.482421875, -1.1708984375, 0.1658935546875, -0.56005859375, -1.154296875, 0.5615234375, 2.021484375, -1.103515625, -0.52490234375, -0.53369140625, 0.54443359375, 0.83154296875, 0.1646728515625, -1.7412109375, -1.1064453125, 0.1324462890625, -0.192138671875, 0.98388671875, -0.509765625, -1.3193359375, -0.71728515625, 0.473388671875, -0.2421875, -0.386962890625, 1.451171875, 0.474365234375, 0.91455078125, 2.498046875, -1.6513671875, 0.8505859375, -0.44091796875, -0.88330078125, 0.465576171875, 0.56640625, -1.96484375, -0.962890625, -0.2208251953125, -0.1923828125, 0.1763916015625, -1.373046875, 0.2052001953125, 1.2392578125, -1.5380859375, 0.0022220611572265625, 0.98583984375, -1.9111328125, -0.044647216796875, 0.1217041015625, 0.6376953125, 0.640625, -0.48779296875, 0.66796875, 0.34326171875, 0.30908203125, 1.7060546875, 1.4609375, -0.297119140625, 0.214111328125, 1.5771484375, 0.8916015625, 0.51025390625, 1.5673828125, -0.4345703125, 0.451416015625, -0.90087890625, 0.69091796875, 0.7001953125, 0.1451416015625, -0.95849609375, -0.92578125, -1.1669921875, -0.08648681640625, -0.63232421875, -0.491943359375, 1.544921875, 0.051788330078125, 0.2220458984375, -1.302734375, -0.280517578125, 2.197265625, 0.1512451171875, 0.453857421875, 0.39990234375, 1.07421875, 0.7421875, 0.52001953125, 0.7646484375, 0.409912109375, 1.728515625, 1.056640625, -0.290283203125, 1.2978515625, 0.1905517578125, -0.525390625, 0.274658203125, 3.0, -0.454833984375, 1.0341796875, 0.303466796875, 1.0712890625, 2.01953125, 0.16162109375, -1.384765625, -0.033966064453125, -0.0826416015625, -0.5576171875, -0.0070953369140625, 0.289306640625, -0.8935546875, -1.4365234375, 0.0987548828125, 0.147216796875, 0.99951171875, 1.2060546875, -1.2744140625, 0.4130859375, -0.34228515625, 1.3447265625, -0.5625, 0.1627197265625, 0.02752685546875, 1.71875, -1.2470703125, -2.89453125, -0.326416015625, -1.029296875, -0.039031982421875, 0.283203125, 0.498046875, -0.239990234375, -0.78466796875, -0.1795654296875, -0.5107421875, 1.212890625, 0.226318359375, 0.1182861328125, -0.1353759765625, 1.1025390625, -1.0361328125, -0.36083984375, -0.82275390625, 1.8798828125, -0.01015472412109375, 0.50439453125, 0.369384765625, 0.943359375, -0.160888671875, -0.99267578125, 0.51416015625, 0.1917724609375, 1.4716796875, 1.0234375, 0.0645751953125, -0.84033203125, -1.0859375, 1.05078125, -1.607421875, -0.135009765625, 0.498291015625, 0.603515625, -0.966796875, -0.14501953125, -0.78515625, 0.2197265625, -0.10491943359375, -0.5517578125, -0.30859375, 1.068359375, -0.172607421875, -1.013671875, 0.68994140625, 1.0224609375, 0.52392578125, -0.58203125, 0.096435546875, -1.6689453125, 0.76806640625, 0.92333984375, 2.0390625, 1.0966796875, 0.58740234375, -0.150146484375, 0.420654296875, -1.333984375, 1.28125, -0.478759765625, -1.5107421875, 1.2236328125, 1.2900390625, 1.720703125, 0.322021484375, -0.2166748046875, -0.453369140625, 0.0098724365234375, 0.020660400390625, 1.1630859375, 2.55859375, 0.468505859375, -1.595703125, -1.1611328125, -2.5078125, 1.3291015625, 0.235595703125, 0.66455078125, -0.006229400634765625, -0.147216796875, -1.087890625, -0.56689453125, -0.0635986328125, 0.77978515625, -2.6171875, -0.86328125, -0.86279296875, -0.4150390625, 1.9228515625, -0.5673828125, 0.677734375, -0.329833984375, 0.097412109375, -0.947265625, 0.31591796875, -1.9091796875, -1.3212890625, 0.56298828125, -0.79736328125, 0.8681640625, -0.34521484375, 0.0275421142578125, -0.04974365234375, -0.390625, 0.301025390625, 0.458984375, 0.15185546875, -0.544921875, -0.63623046875, 1.2412109375, -1.720703125, -0.0280303955078125, 0.78466796875, 3.419921875, -0.3974609375, 1.544921875, -0.99853515625, -0.95654296875, -1.158203125, -0.72412109375, 1.5048828125, -0.08892822265625, -0.09075927734375, -0.1546630859375, 0.56201171875, -1.1337890625, 1.005859375, -0.055206298828125, -1.9541015625, 0.0270843505859375, 0.52099609375, 0.93603515625, 0.337890625, -0.47021484375, -1.7998046875, -0.7451171875, 0.61962890625, -0.0784912109375, 0.5224609375, -1.4111328125, 0.151611328125, 0.055023193359375, 1.109375, -0.387939453125, 0.248779296875, 0.345947265625, -1.087890625, -1.0712890625, 0.47998046875, -1.7578125, -1.6650390625, 0.84814453125, 0.9365234375, 1.193359375, 0.38134765625, -0.65966796875, -0.708984375, -1.5263671875, 0.78271484375, 0.56005859375, -0.14794921875, -0.1595458984375, 0.68603515625, -1.6787109375, 0.365234375, 0.458740234375, -0.63720703125, -0.396240234375, 0.14697265625, 0.006134033203125, -0.78125, -0.431396484375, -0.1190185546875, 2.21875, 0.2841796875, 0.52197265625, 0.236328125, 0.09930419921875, 0.5654296875, -0.3603515625, -1.6240234375, 0.52587890625, 0.22119140625, 0.72021484375, 0.35888671875, 0.05419921875, -0.391845703125, 0.296875, -1.455078125, -0.38671875, -1.81640625, -0.81787109375, -0.08355712890625, -1.91015625, -0.85595703125, -1.3232421875, 0.333740234375, 0.5029296875, 1.150390625, -0.2099609375, 0.97021484375, -0.392578125, -0.1619873046875, -1.1591796875, 0.395751953125, 1.6982421875, -1.2021484375, -0.2078857421875, 0.307373046875, 0.0025730133056640625, 0.119873046875, -0.389404296875, -1.240234375, -0.178955078125, -0.59765625, 0.59326171875, 1.58203125, -1.4736328125, 1.337890625, -1.7333984375, -0.00743865966796875, -0.468017578125, 0.09771728515625, -0.7841796875, -0.27294921875, 0.465576171875, -0.1973876953125, -0.3330078125, -0.114013671875, 1.1611328125, -0.38720703125, 1.744140625, 0.005645751953125, -1.2421875, 0.73828125, -1.1689453125, 2.1875, -0.438232421875, -0.3408203125, -0.1685791015625, 1.041015625, 0.51953125, 0.383544921875, -0.58740234375, 1.4013671875, 1.298828125, -0.39794921875, 1.2724609375, 1.12109375, 0.63916015625, -0.12939453125, -0.54736328125, -0.91015625, 0.272216796875, -0.2122802734375, 2.05078125, -0.61669921875, -0.525390625, -0.27783203125, -1.3984375, -0.95263671875, 0.6982421875, -1.126953125, 1.181640625, 1.1689453125, 0.9921875, -1.2919921875, -0.13720703125, -0.3896484375, -1.2568359375, 0.50439453125, -2.482421875, 1.216796875, 0.11962890625, 0.640625, -0.08660888671875, 0.1468505859375, 0.54150390625, 2.900390625, 1.6630859375, 1.0703125, -0.1973876953125, -2.580078125, -0.00823211669921875, 2.3359375, 0.58740234375, -0.88916015625, -1.6513671875, 0.81787109375, 0.74267578125, 1.166015625, 0.19873046875, 0.416015625, -0.0131378173828125, 1.28125, 1.74609375, 0.247802734375, -0.305908203125, -0.015655517578125, -0.6064453125, 1.0556640625, 0.11834716796875, 0.67333984375, -0.8349609375, 0.75244140625, 1.3798828125, -0.2015380859375, 0.400634765625, 0.7109375, 1.568359375, 3.30859375, 0.4013671875, 0.2215576171875, -0.07562255859375, -1.408203125, 0.689453125, -0.07183837890625, -0.87744140625, 2.3984375, 0.08477783203125, 0.11376953125, -0.2039794921875, 1.236328125, -0.240966796875, 0.52001953125, -1.3974609375, -0.07379150390625, 1.4462890625, 0.7841796875, -0.87841796875, -0.84326171875, 1.2841796875, -1.921875, -1.056640625, 0.8291015625, 1.740234375, 0.99560546875, -0.7373046875, 1.828125, -1.0966796875, -0.1309814453125, -0.321533203125, -1.939453125, 0.767578125, -0.93212890625, 0.322021484375, -1.23828125, -0.71337890625, 0.9677734375, -0.2108154296875, 1.8505859375, 0.65673828125, -0.79443359375, -1.208984375, -0.97314453125, 1.572265625, 0.25634765625, 0.05364990234375, 0.0810546875, 0.71337890625, -1.865234375, -0.479248046875, -0.859375, 0.202392578125, -1.7451171875, -1.849609375, -2.4921875, 0.225341796875, 0.478759765625, 0.445556640625, 0.73095703125, 0.82421875, 0.919921875, 0.15625, -0.231689453125, -0.43603515625, 0.57421875, -1.8251953125, 0.5791015625, 0.01082611083984375, -2.09375, 2.404296875, 0.10113525390625, -0.591796875, 1.1279296875, 0.1923828125, -0.051605224609375, 1.0205078125, 0.24609375, 0.1796875, -1.7998046875, 2.125, -0.252197265625, -0.57958984375, 0.8232421875, 0.33154296875, 2.23046875, 0.869140625, 0.87744140625, -2.1484375, -1.1201171875, -2.70703125, -1.7890625, -2.7578125, 0.71826171875, 1.837890625, 0.7216796875, 1.462890625, 0.15185546875, 1.265625, 2.7734375, 0.439208984375, -0.611328125, -0.495361328125, -1.337890625, -0.298828125, -0.1181640625, -0.85400390625, 0.83251953125, 0.615234375, 1.509765625, -0.70751953125, 0.10601806640625, 0.97314453125, 1.0869140625, 0.85205078125, 0.32470703125, -0.1318359375, -0.2939453125, -0.2110595703125, -2.623046875, -0.5615234375, 1.158203125, 0.099853515625, 1.9931640625, -0.2210693359375, -0.229736328125, 0.10931396484375, 0.556640625, 1.353515625, -0.93115234375, -1.109375, -0.2135009765625, -0.2083740234375, -0.4521484375, 0.463623046875, 0.2320556640625, 1.64453125, -0.35400390625, -1.9365234375, 0.314453125, -0.50537109375, 1.19921875, -0.6591796875, -0.90380859375, 0.385009765625, -0.0303192138671875, -0.52783203125, -0.71923828125, -0.01363372802734375, -1.2177734375, -0.06024169921875, 0.66357421875, -0.1583251953125, 2.689453125, -3.1953125, -0.361083984375, 1.203125, -0.27001953125, -0.1478271484375, -0.65185546875, -1.1591796875, 1.7626953125, 0.65869140625, -1.048828125, -0.94580078125, -0.869140625, -3.115234375, 0.890625, 1.041015625, 0.56689453125, 1.7880859375, 1.189453125, -1.7578125, 1.0078125, -1.3583984375, -0.27001953125, -0.357177734375, 1.7958984375, -0.1251220703125, -1.0498046875, -1.080078125, -1.2646484375, -0.344970703125, -0.1500244140625, 0.58837890625, -0.0167694091796875, 0.86083984375, -0.06610107421875, 0.7353515625, 0.27685546875, 0.69775390625, 0.93212890625, 0.09912109375, -0.9130859375, -0.9052734375, 0.345703125, -1.591796875, 0.3193359375, 0.873046875, 0.0487060546875, 0.390380859375, 0.02935791015625, -1.1962890625, 1.0361328125, -0.806640625, 0.2724609375, 0.20166015625, -1.3798828125, 1.552734375, -2.55859375, -0.10418701171875, -1.099609375, 2.287109375, -0.130615234375, 1.498046875, -0.10791015625, -0.313720703125, 1.48828125, -0.445068359375, -1.3681640625, -0.73486328125, -1.365234375, 0.5224609375, -0.318359375, -0.9091796875, 0.460205078125, -0.153564453125, -1.6748046875, -0.2049560546875, 1.1943359375, -1.19921875, -0.80322265625, -0.234130859375, 0.166015625, 1.12109375, -0.76513671875, -2.720703125, 0.123046875, 0.1256103515625, -0.275390625, 1.921875, -0.6640625, -0.31298828125, 0.77734375, -0.853515625, 1.4033203125, 0.09478759765625, -0.42138671875, -1.021484375, -0.07916259765625, 0.75830078125, 0.541015625, -1.1318359375, 0.78125, -0.31591796875, -1.3935546875, 0.61865234375, -1.0859375, 0.188232421875, 0.2252197265625, 0.28369140625, 0.1251220703125, -1.6875, -0.60595703125, -0.0010004043579101562, -0.8505859375, -0.283935546875, -1.8603515625, 0.25732421875, -1.6708984375, 0.2122802734375, -0.64453125, -1.19921875, 0.1275634765625, 1.4609375, 0.7392578125, 0.428466796875, 0.7021484375, -0.2060546875, -0.83642578125, 0.388427734375, 0.85009765625, -0.50634765625, -0.8017578125, -0.2462158203125, 0.09356689453125, 0.41064453125, -0.052734375, -0.0555419921875, -0.39599609375, 0.311279296875, -0.074462890625, -1.75390625, -0.662109375, 0.07733154296875, 0.031890869140625, -0.78369140625, 0.269287109375, 0.4775390625, -1.2509765625, -1.185546875, 0.1824951171875, 0.2073974609375, -1.7919921875, 3.419921875, 0.55712890625, 1.4697265625, -0.343505859375, -0.1480712890625, -0.30810546875, 0.447509765625, 1.505859375, -0.7158203125, -0.3330078125, -1.3046875, -0.126953125, -0.352294921875, 0.6669921875, 0.077392578125, -2.322265625, -0.77099609375, -0.04669189453125, 0.57080078125, 0.52294921875, 0.935546875, -0.50390625, -0.11798095703125, 0.0295562744140625, 2.1015625, -1.3486328125, 0.3671875, 0.14453125, -2.361328125, -1.8984375, 1.0927734375, -0.1009521484375, 0.66259765625, -0.38916015625, -0.5888671875, -0.261962890625, -0.74755859375, -0.11236572265625, 0.285400390625, -0.3076171875, -1.150390625, -0.7802734375, -0.7158203125, -1.529296875, 0.1961669921875, -0.296875, 0.059844970703125, -1.2080078125, 1.212890625, 0.61328125, -0.7333984375, 1.716796875, 1.9267578125, 0.6396484375, -1.609375, -0.64794921875, 1.984375, 0.94091796875, 0.59814453125, 0.84326171875, 0.6484375, 1.1396484375, -0.76171875, 0.182861328125, -0.08551025390625, -0.402099609375, 0.30322265625, -0.27197265625, 1.0302734375, 0.77099609375, 0.1007080078125, -1.693359375, -0.8984375, 0.06597900390625, -0.1534423828125, -1.7470703125, -0.244873046875, -0.70703125, 0.72900390625, -0.09979248046875, 0.03753662109375, -1.458984375, -0.525390625, -2.376953125, -1.2578125, 0.6279296875, 0.278076171875, -0.0880126953125, 0.0312042236328125, 1.0966796875, -1.34765625, 0.422119140625, 0.2401123046875, 0.1536865234375, 0.34716796875, 0.1898193359375, 0.422607421875, 1.650390625, 0.99169921875, -0.66015625, -0.259521484375, 0.38330078125, -1.193359375, -1.1201171875, -0.1259765625, -1.9873046875, 0.021240234375, 0.048736572265625, 1.447265625, -1.662109375, -2.4296875, -1.0810546875, 0.95654296875, -1.0361328125, 2.04296875, 0.43896484375, -2.080078125, 0.5654296875, 1.7333984375, -0.53271484375, 0.666015625, 1.2578125, 0.67529296875, 1.166015625, 1.0224609375, -1.4853515625, 1.4375, -0.6650390625, -0.401123046875, 0.66796875, -1.7646484375, 1.3330078125, 0.261474609375, 0.10186767578125, -0.86181640625, -0.7666015625, -0.61669921875, 1.26953125, 1.783203125, 1.1962890625, -0.90576171875, 0.9150390625, 0.21240234375, -0.4990234375, 1.13671875, 0.06817626953125, -1.119140625, 0.2386474609375, -0.3916015625, 0.54736328125, -0.8251953125, -1.9287109375, -0.18798828125, 0.22509765625, -0.1993408203125, 0.196044921875, 0.489990234375, -0.12127685546875, -1.0615234375, -0.2222900390625, -0.49658203125, -0.23046875, 0.9208984375, -0.22412109375, 0.1424560546875, -0.1767578125, -1.1611328125, -1.1728515625, -0.112548828125, -0.0031528472900390625, -0.8564453125, 1.9326171875, 0.193115234375, -0.111083984375, 0.70849609375, 0.322265625, -0.615234375, 1.484375, 1.333984375, -1.158203125, -1.3857421875, -1.0869140625, -1.748046875, -0.36962890625, 0.7158203125, -0.26708984375, 0.239013671875, 1.2724609375, 1.7578125, -0.8203125, -0.31591796875, 0.953125, 0.22265625, 1.2060546875, -1.9306640625, -1.33203125, -0.54345703125, -0.490234375, 0.95068359375, 1.005859375, -1.263671875, 0.4013671875, -0.4130859375, 0.40283203125, -0.325927734375, -0.72314453125, 0.365234375, -1.251953125, -0.444091796875, 0.38427734375, 0.222900390625, -0.237060546875, -0.51708984375, 2.52734375, 2.236328125, 0.83837890625, 0.33740234375, 1.7431640625, 0.63134765625, 0.7783203125, -1.8828125, -1.21484375, 1.541015625, 0.6220703125, 0.62451171875, -0.25439453125, 0.5107421875, 0.82958984375, 1.318359375, 1.4306640625, 0.85498046875, -0.53369140625, -1.2158203125, -1.9892578125, -0.947265625, 0.70947265625, 0.66748046875, 0.338134765625, 1.744140625, 0.239013671875, -0.0086669921875, 0.5966796875, -0.1319580078125, -0.3603515625, -1.7548828125, -0.367431640625, 0.59130859375, -0.309814453125, -0.970703125, -0.315673828125, -1.6962890625, -0.7802734375, -1.4423828125, -1.150390625, 2.337890625, 0.9638671875, 0.485107421875, -0.2724609375, 0.128173828125, 0.53759765625, -0.29150390625, -0.7861328125, -0.351318359375, 0.472900390625, 0.1416015625, -0.267822265625, 0.77197265625, -0.72119140625, 0.92822265625, 0.4794921875, -2.3828125, 1.3642578125, 0.6796875, -0.10137939453125, 1.302734375, -0.270263671875, 0.94921875, -0.73486328125, 0.80908203125, -0.036163330078125, -0.4833984375, 0.8017578125, -1.615234375, -0.330078125, 0.658203125, -0.98876953125, -1.013671875, -0.375244140625, -1.9228515625, 0.359130859375, 0.5947265625, 0.7861328125, -1.015625, -0.1334228515625, -0.482177734375, 0.1689453125, -0.908203125, 0.2071533203125, -0.18310546875, 0.88623046875, 0.1787109375, 0.6484375, -1.1962890625, 0.11517333984375, 0.802734375, -2.673828125, 0.80615234375, -0.390380859375, 0.5869140625, 0.413818359375, 1.23046875, -0.92578125, 1.7529296875, 1.2568359375, 0.50048828125, 2.56640625, -0.372802734375, 0.80859375, 0.08917236328125, 2.330078125, -0.6796875, -1.40625, 0.68701171875, 0.345947265625, -1.751953125, 0.269775390625, 0.2734375, -2.373046875, -0.42333984375, -0.21142578125, 1.7333984375, -1.7060546875, 0.54052734375, -0.0008740425109863281, -0.9560546875, -0.047760009765625, -0.5419921875, -1.20703125, 1.06640625, -0.89990234375, -0.55859375, -0.16162109375, 0.76806640625, 1.0166015625, -0.64697265625, -1.6220703125, 1.84375, -0.9833984375, 0.118896484375, -0.283935546875, 0.306396484375, 0.5234375, 2.01171875, 0.342041015625, 1.73828125, 2.845703125, 0.91455078125, -0.029083251953125, 0.400390625, -0.47119140625, 0.3466796875, -1.375, -0.85400390625, -0.1055908203125, -0.4619140625, -1.087890625, -0.69140625, 1.2373046875, 0.2242431640625, 0.42626953125, 0.89892578125, 1.896484375, 0.73876953125, -0.6611328125, 0.46142578125, -0.71826171875, -1.2607421875, -0.1087646484375, -0.8896484375, -0.43603515625, 1.0537109375, 1.4580078125, -0.62548828125, 0.0767822265625, 1.3466796875, -0.1336669921875, -0.6494140625, 0.1981201171875, 0.23779296875, 1.2626953125, -1.0654296875, -0.39453125, -0.34765625, 0.9716796875, -0.285400390625, 1.162109375, 2.255859375, 0.2197265625, -0.77099609375, -0.477783203125, 0.51416015625, 0.01336669921875, -0.425537109375, 0.765625, 0.3837890625, 0.814453125, -1.083984375, 1.1494140625, -0.35888671875, -0.294921875, -0.415283203125, 1.1083984375, 0.53369140625, -0.1712646484375, -0.54541015625, -0.45849609375, 0.060760498046875, 1.755859375, 1.4599609375, -1.6865234375, -0.6240234375, 3.3984375, -1.359375, 0.5107421875, 0.484130859375, -0.3291015625, -0.56982421875, 0.60791015625, 0.91845703125, 0.410888671875, -0.4326171875, 1.3544921875, 0.45068359375, 1.775390625, -0.0799560546875, 0.1827392578125, -0.41650390625, -0.51318359375, -0.01245880126953125, 0.5498046875, -1.4970703125, -1.6337890625, 0.312744140625, 0.7763671875, -0.455810546875, 0.78125, 0.76025390625, 0.07861328125, -1.0361328125, 0.111083984375, 0.54638671875, 0.765625, -1.1669921875, -0.92578125, -0.89794921875, 1.8623046875, 0.1273193359375, 1.0380859375, -0.498046875, 0.3798828125, 1.7900390625, 0.1416015625, -0.0119781494140625, 1.44921875, 0.347900390625, -0.97314453125, 0.357177734375, 0.73828125, -0.094970703125, -0.62451171875, 0.3349609375, -1.4091796875, 0.45458984375, -1.6728515625, 1.203125, -1.615234375, 1.421875, 0.97216796875, -0.9765625, 1.1533203125, -0.86083984375, -1.2294921875, 0.1920166015625, -1.3076171875, -0.6162109375, 1.0576171875, -1.353515625, 0.482421875, 0.5927734375, -1.0546875, -1.47265625, 0.1129150390625, 0.0213470458984375, 0.72802734375, -1.576171875, -2.333984375, -0.88330078125, -0.107666015625, 1.5263671875, 1.001953125, 1.42578125, -0.87841796875, 1.716796875, 1.3095703125, 2.458984375, 1.015625, 2.5078125, 0.11444091796875, -1.419921875, 0.52392578125, -0.75341796875, -1.4013671875, -0.327880859375, -1.0576171875, -0.5439453125, 1.46875, -1.2548828125, -0.6923828125, -0.3388671875, 1.5673828125, -0.70947265625, 0.591796875, 0.1864013671875, 1.625, -0.470458984375, 0.486572265625, 1.501953125, 0.62255859375, -0.97900390625, -0.66259765625, 0.149169921875, -0.09320068359375, 0.41064453125, 0.04327392578125, -0.364501953125, -0.587890625, 0.482666015625, 2.037109375, -0.394287109375, -0.137939453125, -1.1337890625, -0.345458984375, 1.6259765625, -1.0849609375, 0.189697265625, -0.93798828125, -0.8271484375, 0.37109375, -0.1214599609375, 0.1661376953125, 0.77099609375, -1.1015625, 0.67333984375, -0.82666015625, -0.2147216796875, -0.03863525390625, -1.2666015625, 0.274169921875, 0.97705078125, 1.3583984375, -1.123046875, -1.0107421875, -0.414794921875, -0.256103515625, 1.1015625, 0.310791015625, -0.71435546875, -0.81787109375, -0.95263671875, 1.25390625, -0.485595703125, -0.139404296875, 1.076171875, 0.439697265625, -1.501953125, -0.390380859375, 0.28125, 0.99072265625, 1.3447265625, -0.327880859375, -0.004390716552734375, 0.3212890625, -0.68701171875, -0.050262451171875, -0.71875, -0.26513671875, -1.5625, -0.544921875, -1.09375, 0.305419921875, -0.89892578125, -0.6650390625, -3.0390625, -0.53466796875, 0.279541015625, 0.63330078125, -1.349609375, -0.32568359375, -0.0721435546875, 2.30078125, -0.83154296875, -1.513671875, -0.69189453125, 0.1650390625, -0.36669921875, 0.0574951171875, 0.0667724609375, 0.1990966796875, -0.1705322265625, 0.662109375, 0.41455078125, -0.476318359375, -0.9130859375, 1.2548828125, 0.368408203125, -1.443359375, 1.1279296875, -0.5634765625, -0.51953125, 1.447265625, -0.8447265625, 0.5068359375, -0.487548828125, -0.296630859375, 1.1806640625, 1.794921875, 0.8974609375, 0.1441650390625, 0.40185546875, 1.92578125, -0.68359375, -0.2200927734375, -0.2113037109375, 1.4462890625, -1.1494140625, 0.54248046875, 0.291748046875, -1.1162109375, 0.08709716796875, -0.86328125, -0.61962890625, 0.0863037109375, 0.41845703125, -1.095703125, 1.4140625, 0.27490234375, -0.478271484375, -0.9921875, -0.447509765625, -0.0594482421875, 0.10888671875, -0.466796875, -1.5869140625, -1.0126953125, -1.31640625, 2.09375, 1.62109375, -0.397705078125, 0.6484375, 0.50048828125, 0.050201416015625, -0.452392578125, -1.2529296875, 0.9169921875, -0.62109375, 0.1822509765625, -1.2822265625, -0.004749298095703125, 0.68994140625, 0.7900390625, 0.436279296875, 1.8046875, -1.9677734375, 2.673828125, -2.150390625, -1.3876953125, 0.3681640625, 0.2227783203125, 0.7529296875, 0.2120361328125, 0.52587890625, -0.246826171875, 0.88134765625, -0.3857421875, 1.0693359375, 0.7568359375, -0.7978515625, -0.23486328125, -0.33056640625, 0.51318359375, 1.9169921875, 0.88330078125, -1.169921875, -2.62109375, -0.0662841796875, -0.685546875, -2.958984375, -0.318603515625, -1.0859375, 0.326171875, -0.54248046875, -1.23046875, -0.72021484375, 1.4609375, -1.19140625, 0.404052734375, -0.257080078125, -0.09417724609375, -1.212890625, 0.86083984375, -0.693359375, 1.0205078125, -1.2939453125, 0.83154296875, -0.44873046875, 1.9453125, -0.92041015625, -1.40625, -0.88818359375, -0.3955078125, -0.7607421875, -0.58349609375, -0.392822265625, 0.04986572265625, -0.42626953125, 1.30078125, 0.5048828125, -1.4462890625, 0.2470703125, 0.2469482421875, -0.47412109375, 0.239990234375, 2.341796875, 0.8935546875, 0.23681640625, -0.92919921875, 2.04296875, 0.1884765625, -1.8720703125, -0.5498046875, -1.1796875, -0.7392578125, 0.64794921875, 1.3232421875, 0.61181640625, -0.07208251953125, -0.229736328125, -0.52734375, 0.49560546875, -1.263671875, -0.82421875, 0.7958984375, -0.358642578125, 0.67138671875, -1.408203125, 0.478759765625, 1.94921875, -0.77197265625, 2.861328125, -0.2032470703125, -0.79248046875, -1.2724609375, 1.03515625, -0.004009246826171875, 0.53564453125, 0.9599609375, -0.72216796875, 0.51171875, -1.212890625, -0.373779296875, 0.4990234375, 0.301025390625, 0.89404296875, 0.140380859375, -1.1396484375, 0.9794921875, -1.0205078125, 0.91943359375, -0.1783447265625, -0.044647216796875, -0.6201171875, 3.80078125, -0.3466796875, 0.1533203125, 0.47412109375, -0.31201171875, -0.83154296875, 0.74609375, -0.291748046875, 0.24267578125, -0.03363037109375, -1.01171875, -1.7294921875, 0.95361328125, 1.4111328125, 2.048828125, 0.783203125, -0.418212890625, -0.625, 0.65673828125, -0.369140625, -1.3056640625, 0.202392578125, -0.6796875, -0.580078125, 0.485107421875, -0.5673828125, -0.99365234375, -1.4150390625, 1.447265625, -1.162109375, 0.51904296875, -1.576171875, 0.353271484375, -0.27099609375, 0.8125, -0.325927734375, 0.35205078125, -0.9814453125, -2.13671875, -1.4755859375, -0.04986572265625, -0.91845703125, 0.85791015625, 0.6669921875, -0.29833984375, 0.12249755859375, 1.2412109375, -0.9208984375, -2.197265625, -1.1640625, 0.2457275390625, 0.490966796875, -0.1422119140625, 1.716796875, 0.1490478515625, -0.447998046875, 0.98486328125, -1.3740234375, -0.4609375, 0.81494140625, -1.0302734375, 0.213623046875, -0.71923828125, 0.82177734375, -0.47216796875, -0.5419921875, -0.39501953125, -1.697265625, 0.89453125, 0.8046875, -1.0986328125, -0.25439453125, 0.60791015625, 0.397216796875, 0.4912109375, 0.62451171875, 0.59130859375, -1.3349609375, -0.68115234375, -0.72900390625, -1.41796875, 0.9970703125, 1.84375, 0.446044921875, 1.25, -1.1611328125, -0.41552734375, -2.908203125, -1.4794921875, -0.77734375, 2.025390625, -0.4267578125, 1.2783203125, -0.7529296875, -0.2568359375, 0.1341552734375, -0.2073974609375, 0.28271484375, -0.110107421875, -0.4384765625, 0.1845703125, -0.040069580078125, -1.169921875, 0.47021484375, 0.46337890625, -1.42578125, 0.4990234375, 1.0703125, -1.68359375, -0.021820068359375, -0.041717529296875, -0.71875, 0.57421875, -0.1573486328125, 1.09765625, -0.119140625, -1.8046875, 0.60498046875, -0.44091796875, -0.55615234375, -0.08642578125, -0.12261962890625, -1.0771484375, 1.0703125, 1.0205078125, -1.3212890625, -0.0738525390625, 1.42578125, -0.369873046875, 1.6689453125, -0.75634765625, -1.427734375, -1.279296875, -0.83544921875, -1.138671875, -1.8193359375, -1.3544921875, -1.3251953125, 1.1435546875, -0.88671875, 0.1734619140625, 0.98046875, 0.81884765625, 1.32421875, 1.0751953125, 0.137939453125, -0.04351806640625, -1.8466796875, -0.4853515625, 0.9228515625, -0.186767578125, 0.779296875, 0.98388671875, 1.53125, 1.1025390625, -0.75048828125, 0.82666015625, 1.126953125, -0.625, 0.450439453125, 0.180419921875, 0.31494140625, 0.66748046875, 4.01171875, 0.13037109375, -0.85400390625, 0.276123046875, -0.25, 1.2451171875, 2.5625, -0.1617431640625, 0.35595703125, 0.057098388671875, -0.4384765625, 0.009063720703125, -0.404541015625, -0.1651611328125, 1.3515625, 1.1181640625, 2.806640625, -2.181640625, 1.24609375, 0.5078125, 0.00887298583984375, -0.3896484375, -0.51171875, 0.5205078125, 0.6005859375, -0.0916748046875, 0.95458984375, -0.92578125, 0.859375, 1.3828125, -0.414306640625, 0.88818359375, 1.173828125, 0.340087890625, 1.1650390625, 0.7080078125, 0.5087890625, -0.247314453125, 0.49609375, 0.8076171875, 1.5576171875, -0.84619140625, 0.05609130859375, 0.55517578125, 0.49609375, 1.8759765625, 0.394287109375, -1.314453125, -2.705078125, 1.3798828125, 0.69775390625, 0.1461181640625, 0.72412109375, 0.1397705078125, 0.8974609375, 0.310302734375, -0.5234375, 0.414794921875, 1.185546875, -0.6083984375, -0.427978515625, 1.3505859375, 0.7509765625, 0.56689453125, -0.67138671875, -1.0625, -0.1951904296875, -0.301513671875, 0.459228515625, -1.552734375, -0.63623046875, -0.70263671875, 0.425537109375, -0.85302734375, -1.5439453125, -1.013671875, -1.501953125, 0.1436767578125, -1.611328125, 1.015625, -0.1302490234375, -1.41015625, -1.6376953125, -0.06622314453125, -0.82666015625, 0.33935546875, -0.814453125, 0.775390625, 0.78173828125, 0.4228515625, 0.78564453125, 0.97412109375, -1.7333984375, -0.51611328125, -1.23828125, -0.68603515625, -0.287109375, -0.98291015625, 0.525390625, -0.3447265625, 1.3662109375, -1.3837890625, 0.71630859375, -1.2060546875, -0.44677734375, -1.5537109375, -0.348876953125, 1.7900390625, 0.25390625, 0.350830078125, -0.6396484375, 0.8349609375, -0.160400390625, 0.13134765625, 0.65625, 1.2548828125, 1.1884765625, -0.2261962890625, 1.96484375, 1.00390625, -0.98046875, 0.365478515625, -0.89013671875, -0.348876953125, 1.4443359375, 0.400390625, 1.0107421875, 1.0693359375, 0.408447265625, 0.91845703125, 0.0537109375, -0.7333984375, -0.68115234375, 0.60791015625, -0.44775390625, -0.76513671875, -0.83642578125, -0.1395263671875, -0.119873046875, -1.69140625, 0.3046875, -0.5439453125, -0.064697265625, -0.377197265625, 1.98046875, 1.431640625, -1.3720703125, -1.6337890625, -0.2822265625, 1.79296875, -1.6337890625, 0.8876953125, -1.6943359375, -0.0216827392578125, 0.1370849609375, 0.744140625, 0.244873046875, -0.513671875, -0.451416015625, 1.0361328125, -1.671875, 0.456787109375, -0.88623046875, 0.27294921875, -0.200439453125, -0.56494140625, 0.396240234375, -1.0869140625, 1.1220703125, -0.6376953125, 0.71826171875, -1.1044921875, -1.3232421875, 0.0556640625, -0.000774383544921875, -3.13671875, -0.2127685546875, 1.759765625, 0.420166015625, -2.755859375, -1.5751953125, 0.03497314453125, 0.3818359375, -0.1356201171875, 1.0107421875, -0.1724853515625, 0.95703125, -1.7412109375, -0.281494140625, 0.54345703125, -0.01861572265625, -0.84033203125, 1.5595703125, 0.1976318359375, 0.3720703125, -0.94677734375, -1.7568359375, 1.33984375, 0.0428466796875, -1.375, 0.3603515625, -1.3955078125, -0.483154296875, -0.41015625, 1.1171875, 0.046112060546875, -0.293212890625, -0.94970703125, 0.10113525390625, 1.875, -1.02734375, 0.68310546875, -1.5068359375, -0.1737060546875, -0.57470703125, -0.93115234375, -0.07440185546875, -0.80810546875, -1.0341796875, 0.7470703125, 0.1956787109375, 0.1697998046875, -1.2119140625, -0.60107421875, 0.25634765625, 1.46875, -1.072265625, -0.223876953125, 0.2103271484375, 0.89404296875, 0.79052734375, -0.0272674560546875, 1.1728515625, 0.2362060546875, -0.57568359375, 0.308837890625, 0.57275390625, -0.00036334991455078125, 1.548828125, -1.900390625, 0.927734375, -1.3017578125, 0.435791015625, 0.1331787109375, 0.578125, -1.51171875, -0.321044921875, 0.84033203125, 0.6982421875, 0.5439453125, -0.2320556640625, 0.142333984375, 1.619140625, -0.1326904296875, 1.6591796875, -0.4501953125, -0.458251953125, 0.41015625, -0.1513671875, -0.25537109375, 0.0113372802734375, 1.4619140625, 1.1630859375, -0.76318359375, 0.78125, -0.090576171875, -0.28857421875, -1.275390625, 0.82861328125, 0.9970703125, -1.279296875, 0.4306640625, 1.3486328125, 1.6201171875, 1.6337890625, 0.52685546875, 1.033203125, 0.04132080078125, 0.1806640625, -0.10137939453125, 0.09588623046875, 0.1363525390625, -0.196533203125, 2.4140625, 0.401611328125, -1.0185546875, 1.26953125, 0.7392578125, 1.62109375, 0.8408203125, -1.2958984375, -0.16748046875, 0.5732421875, -1.1884765625, -0.80029296875, 1.0771484375, -0.484375, -0.25244140625, -0.274658203125, 1.4873046875, 0.57861328125, 0.7255859375, -0.50439453125, 2.705078125, -0.06988525390625, 1.4970703125, 0.034942626953125, 0.01549530029296875, -0.8447265625, -0.391357421875, -0.90771484375, -0.5751953125, 0.348388671875, -1.46484375, -0.861328125, -0.87646484375, -0.340576171875, -0.0653076171875, -0.9697265625, -0.626953125, -0.053863525390625, -2.537109375, -0.73291015625, 0.65576171875, -1.5400390625, 1.6572265625, 0.68359375, -0.5400390625, -1.6787109375, -0.77587890625, 0.1697998046875, -0.64013671875, 0.298583984375, 0.055145263671875, 1.873046875, -0.440185546875, 2.16015625, -0.8955078125, -0.87060546875, 1.4814453125, 0.9873046875, -0.94482421875, 0.869140625, 2.48046875, 1.3544921875, -0.8984375, 0.09918212890625, -0.6201171875, 0.93994140625, -0.4189453125, 0.191650390625, 1.0673828125, 1.4521484375, -0.2225341796875, 1.482421875, -0.1136474609375, -0.1026611328125, -0.4072265625, 1.091796875, -1.30859375, 1.3798828125, 0.7646484375, 0.441650390625, 1.46875, -0.1990966796875, 1.3212890625, 0.57568359375, 0.32568359375, -1.060546875, 0.352294921875, -0.2086181640625, -1.80078125, 0.374267578125, 0.291259765625, 0.2418212890625, 0.1658935546875, -0.400390625, 0.2161865234375, -0.43798828125, -1.5322265625, -0.287353515625, 2.26171875, 1.0341796875, 0.71728515625, -0.81005859375, -1.5654296875, 0.74951171875, 0.38232421875, 0.360107421875, 0.197265625, -1.3779296875, -0.412109375, -0.66064453125, 0.84033203125, -0.340576171875, -0.2275390625, 0.5546875, 1.765625, 0.50732421875, -0.583984375, 0.99072265625, 0.260498046875, -0.483642578125, -1.70703125, -0.90771484375, -1.6240234375, -0.0936279296875, -1.2236328125, 1.009765625, -1.576171875, 0.239013671875, 0.29150390625, 0.374267578125, -0.097900390625, -0.4453125, 0.50927734375, -1.6640625, -0.54931640625, -0.056488037109375, -0.1339111328125, -0.27490234375, 0.7685546875, -0.6962890625, -0.5712890625, 1.365234375, -0.484619140625, -0.1710205078125, 1.1337890625, -0.56787109375, 0.264892578125, -1.052734375, -0.410888671875, -1.435546875, -0.158203125, 0.70703125, 1.078125, 0.83642578125, 0.71728515625, 0.128662109375, -1.0810546875, -1.0888671875, -0.016571044921875, -0.53125, -0.5068359375, 0.50341796875, -0.0367431640625, 0.748046875, 1.25390625, -0.373291015625, 0.1390380859375, 0.91357421875, 1.1611328125, 1.5146484375, -0.71826171875, 2.33984375, 0.032073974609375, -0.303466796875, 0.47021484375, 1.74609375, -0.319091796875, 0.3720703125, 0.252197265625, -0.04168701171875, 0.98095703125, -1.013671875, 0.8994140625, -0.408203125, -0.09588623046875, -0.7578125, -1.8984375, 0.51953125, 1.7880859375, -0.385986328125, -0.9130859375, 0.8203125, -0.0418701171875, -1.2001953125, -0.1541748046875, 0.91162109375, 0.472900390625, -0.65966796875, 0.1983642578125, 0.92529296875, -0.492919921875, 0.24853515625, 0.3193359375, -0.441162109375, -1.5947265625, -0.4521484375, -0.97802734375, -0.72998046875, 0.340576171875, -1.0771484375, -2.037109375, 0.2900390625, -0.09454345703125, -0.1822509765625, -0.12457275390625, -1.1630859375, 0.48681640625, -0.10345458984375, 0.8935546875, -0.02716064453125, -1.0185546875, 0.325439453125, 2.505859375, -0.96337890625, -0.97021484375, -0.10601806640625, -0.410400390625, 1.3544921875, 1.146484375, -0.83740234375, -1.26953125, 1.7861328125, 0.5419921875, -0.356201171875, 0.4365234375, 1.4365234375, 0.92822265625, -0.1824951171875, 0.1710205078125, -0.347900390625, 0.68994140625, -0.3671875, -1.0869140625, 0.332275390625, 1.22265625, -1.30078125, -0.203857421875, 3.115234375, -1.3759765625, -0.5615234375, -0.95263671875, -0.984375, -0.63232421875, 0.74169921875, 0.0232391357421875, -0.2061767578125, 0.36328125, -0.0058746337890625, -0.467041015625, -1.009765625, -0.212646484375, -0.9208984375, 0.7998046875, 0.494384765625, -0.91357421875, 0.50390625, 0.93505859375, 1.9208984375, 0.09661865234375, -0.292236328125, 1.2353515625, 0.173828125, -0.75048828125, -0.060943603515625, 0.7548828125, -0.2646484375, -0.44873046875, 0.0972900390625, 1.724609375, -0.50341796875, -0.048858642578125, -0.376708984375, -1.95703125, -0.1661376953125, 0.37255859375, -0.9912109375, -1.2392578125, 0.033172607421875, 2.484375, -0.23974609375, -0.5634765625, -0.6396484375, 0.30126953125, 0.07183837890625, -1.1669921875, 1.0361328125, 1.4765625, 0.68017578125, 1.458984375, 0.359130859375, 0.50537109375, 1.2998046875, -0.441162109375, 1.2705078125, 1.0224609375, 0.5439453125, -1.00390625, -0.9384765625, 0.63720703125, 0.810546875, -0.5087890625, 0.5224609375, -0.79833984375, 0.436767578125, -1.5947265625, 0.990234375, -1.15625, -2.263671875, -1.228515625, 0.8525390625, -0.2080078125, -1.2958984375, 0.0809326171875, 0.05621337890625, 1.046875, 1.1953125, -0.2919921875, 2.33984375, 1.03125, -0.261962890625, -1.7607421875, -0.5107421875, -1.33984375, -0.43310546875, 0.345703125, -0.41455078125, -1.263671875, 0.27197265625, 0.18701171875, 0.35986328125, 0.83642578125, 1.5810546875, 1.763671875, 1.4404296875, -0.87353515625, -0.3017578125, 1.04296875, -0.1134033203125, -1.244140625, -0.1395263671875, -0.9013671875, -0.68115234375, -0.8076171875, -0.86181640625, -0.447998046875, -1.1904296875, -0.64599609375, 0.305419921875, 1.3740234375, -0.68212890625, 1.05859375, -0.383544921875, -0.3837890625, -0.2098388671875, 0.88134765625, -0.1536865234375, -0.237060546875, 1.201171875, 1.0185546875, -0.170166015625, -0.70068359375, -0.252685546875, 0.958984375, -2.0703125, 0.45849609375, 0.689453125, -0.5830078125, -0.9169921875, 0.46533203125, -0.342041015625, -1.0146484375, 0.045867919921875, 0.55078125, -1.884765625, 0.244384765625, 0.260986328125, 0.77880859375, -0.83642578125, 0.401123046875, 0.2744140625, -0.167236328125, -0.6416015625, 1.978515625, -0.26513671875, -0.576171875, -0.450927734375, -0.88232421875, -1.3779296875, 0.67822265625, -1.654296875, 0.55517578125, -0.7080078125, 2.134765625, 0.409912109375, -0.6962890625, -0.473388671875, -0.02056884765625, 0.517578125, -0.6455078125, -0.72265625, -0.263427734375, -1.625, -0.429443359375, -2.55078125, 0.69677734375, -0.11663818359375, 0.440185546875, 1.4296875, 0.60791015625, -0.13330078125, 0.9296875, 0.14013671875, 0.56005859375, 0.5185546875, 1.302734375, -1.0546875, 0.69091796875, -1.0546875, 0.1689453125, -0.1192626953125, 0.111572265625, 0.81494140625, 0.334228515625, -0.728515625, -0.21484375, 0.424072265625, -0.84765625, 0.8388671875, -0.1282958984375, 1.9990234375, 0.56298828125, -1.16796875, -0.32373046875, -1.5693359375, 0.12286376953125, 0.0640869140625, -0.2437744140625, -0.02532958984375, 0.54296875, -0.250732421875, 0.65380859375, 0.53564453125, 0.56591796875, 0.13671875, -2.490234375, 1.5263671875, -0.7138671875, 1.537109375, 0.615234375, -0.6640625, 0.311767578125, 1.2158203125, 2.23828125, 0.72216796875, 0.79248046875, -0.219482421875, 1.4375, 0.2059326171875, -1.03515625, -1.2275390625, -0.1676025390625, -0.984375, 1.4033203125, -0.6630859375, -2.45703125, -0.327880859375, -0.83056640625, 0.48583984375, 0.0789794921875, 0.2607421875, 1.904296875, 0.483154296875, -0.53515625, -0.10272216796875, 0.42919921875, -0.342529296875, 0.6533203125, 2.7109375, 0.07940673828125, -1.04296875, 0.274169921875, 1.421875, -1.91015625, -0.1727294921875, -0.654296875, 0.25146484375, -0.47998046875, -0.7255859375, -0.64208984375, -0.1329345703125, -0.71630859375, -0.2578125, 0.11590576171875, -0.6220703125, -0.95751953125, -1.0302734375, 0.36181640625, -0.26416015625, 0.382080078125, 0.44775390625, -3.115234375, -0.266845703125, 0.86474609375, -1.587890625, 0.96142578125, -0.48046875, 1.1748046875, 0.20458984375, -0.55517578125, 0.313232421875, -0.0162353515625, 1.3427734375, -1.044921875, -0.56103515625, -1.35546875, 0.239013671875, -0.52490234375, 0.362060546875, -1.01171875, -0.4375, 1.1982421875, -0.37646484375, 1.5302734375, 0.77587890625, 0.68798828125, 0.6875, -1.515625, 0.058746337890625, -1.166015625, -0.262451171875, 0.58251953125, 1.1953125, -0.87646484375, 0.697265625, -0.6640625, 0.92578125, 1.7109375, -0.2158203125, -1.2119140625, 2.12890625, 0.78125, 0.751953125, 0.357421875, 0.74755859375, 1.8125, 0.1937255859375, 0.242919921875, -1.3330078125, -1.12890625, -0.888671875, -0.12237548828125, 0.1766357421875, 0.84326171875, -0.232421875, 0.1514892578125, 0.2237548828125, 1.59375, 1.2509765625, 0.302978515625, 0.2529296875, 0.007724761962890625, 0.1883544921875, 0.75439453125, -0.257568359375, 1.138671875, 2.794921875, -1.34375, 0.091064453125, -0.76025390625, -0.2763671875, -0.349365234375, 0.9306640625, -1.501953125, 1.017578125, -0.94482421875, 0.52978515625, 0.456298828125, -0.272705078125, 0.90576171875, 0.88916015625, 0.73583984375, 0.00801849365234375, 0.06390380859375, 0.65185546875, 0.61865234375, 0.495849609375, 0.1766357421875, 0.1961669921875, -1.5, 0.322509765625, 1.0556640625, 0.9228515625, 0.305419921875, -0.38720703125, 0.96630859375, 0.05609130859375, -0.6962890625, -0.78369140625, -0.55517578125, 1.1767578125, -1.09375, -0.48583984375, -0.87451171875, -1.654296875, 0.053955078125, -0.86474609375, 0.8447265625, -1.11328125, -1.2802734375, -0.44189453125, 1.5390625, 0.01092529296875, -0.1265869140625, -0.87646484375, 0.186279296875, 1.1943359375, 0.489990234375, -0.094482421875, 3.626953125, 0.97021484375, 0.16259765625, -0.0657958984375, 0.256103515625, 0.1500244140625, 0.428466796875, -0.11834716796875, -0.509765625, 0.052398681640625, -1.05859375, 0.5732421875, -0.51220703125, -1.111328125, -0.50927734375, 1.39453125, 1.6865234375, 1.576171875, -0.464111328125, -1.787109375, 0.75048828125, -1.44921875, 2.05078125, 0.1282958984375, 0.81787109375, -0.97705078125, -1.166015625, -0.2156982421875, 0.0213775634765625, -2.478515625, -0.2156982421875, -0.464599609375, -1.3564453125, 0.1778564453125, -0.059844970703125, -0.9833984375, 1.7958984375, 0.11669921875, -0.52880859375, -1.40625, 0.397216796875, 0.9208984375, -1.2919921875, -0.69775390625, -1.9208984375, 0.490966796875, 0.7900390625, 0.859375, -2.271484375, -0.08544921875, -0.31689453125, 1.447265625, 1.494140625, -0.552734375, 0.353271484375, 0.1668701171875, -0.1629638671875, -0.78125, -0.12396240234375, -0.2406005859375, -0.650390625, 0.1588134765625, 1.025390625, -1.1455078125, -0.9248046875, -0.77392578125, 1.720703125, -1.7802734375, -0.5830078125, -2.072265625, -1.4189453125, 0.1787109375, 0.52587890625, -0.1700439453125, -0.80908203125, -1.4248046875, -1.1884765625, 1.8232421875, 0.355712890625, 0.57861328125, -0.97509765625, 0.5869140625, -0.194580078125, 0.52001953125, 1.2470703125, 0.81591796875, 0.580078125, -0.0845947265625, 1.0693359375, -0.38037109375, 1.0478515625, 1.2265625, 0.10650634765625, -0.055389404296875, 0.7548828125, -0.08087158203125, 0.43896484375, 0.732421875, -0.268798828125, -1.3173828125, 0.7646484375, 2.015625, -0.135498046875, -1.978515625, 1.4208984375, -0.1373291015625, 0.3701171875, 1.125, 0.489501953125, -0.72265625, -0.378662109375, 1.1494140625, -0.49755859375, -1.1474609375, 0.76904296875, -2.109375, 2.740234375, 1.7841796875, -1.6328125, -0.974609375, -0.64013671875, 0.61767578125, 0.0362548828125, 0.97509765625, -1.5107421875, 1.05078125, 3.349609375, -0.056640625, 0.759765625, 0.63720703125, -2.8125, 0.23828125, 0.5986328125, -1.9619140625, -1.853515625, -1.2685546875, 0.36279296875, -0.69970703125, -0.888671875, -0.7275390625, 0.306640625, 0.64697265625, -0.5498046875, -0.189208984375, -0.266845703125, 0.09429931640625, -1.9453125, -0.50927734375, 0.183837890625, 0.71435546875, -0.1767578125, -0.1729736328125, 0.1793212890625, 0.36767578125, -0.52880859375, 0.947265625, 0.2276611328125, -0.1920166015625, 0.79150390625, -0.82763671875, -1.3623046875, 0.484375, 0.097412109375, 0.47021484375, 0.363037109375, 0.261962890625, -0.37451171875, -0.1240234375, 1.0859375, 0.313720703125, 1.5224609375, -0.87353515625, 0.52197265625, 2.998046875, 0.08575439453125, 0.305419921875, 0.045928955078125, 0.79931640625, 0.1875, -0.164306640625, 0.2457275390625, 1.45703125, 0.491455078125, -0.63427734375, 0.8251953125, 1.0224609375, 1.234375, 1.537109375, 0.58203125, -0.1162109375, 1.5166015625, 0.384033203125, -0.13916015625, 0.06060791015625, -0.0726318359375, 0.9423828125, 1.705078125, -1.2685546875, -0.6298828125, 0.97998046875, 0.08233642578125, -0.62109375, -0.0843505859375, 1.4560546875, 0.198974609375, 0.25439453125, 0.122314453125, 0.56591796875, -1.2939453125, 1.8828125, -1.16796875, 0.59619140625, 0.034423828125, -1.48828125, -0.27783203125, -0.390380859375, 1.1689453125, -0.658203125, 0.224609375, -0.397216796875, -0.3212890625, 1.126953125, 0.62841796875, -1.427734375, -1.330078125, 0.1729736328125, 0.68017578125, 2.091796875, 1.0556640625, -0.414306640625, 1.8134765625, 1.080078125, 1.0888671875, -0.32275390625, -1.46484375, -0.0736083984375, 1.2275390625, 0.402099609375, 1.2353515625, -1.103515625, 0.0291290283203125, 0.89208984375, 0.9482421875, -1.1865234375, -1.7431640625, 0.0791015625, -0.428955078125, 1.32421875, 1.2294921875, 0.63134765625, 0.14501953125, 0.72119140625, -0.51513671875, -0.181396484375, -1.396484375, 0.0352783203125, -1.265625, 0.69873046875, -0.386474609375, -2.3046875, 1.326171875, -0.48779296875, 1.0224609375, -0.998046875, 0.1263427734375, -0.0186920166015625, -0.1812744140625, -0.240478515625, 1.32421875, 0.379638671875, 1.7373046875, -1.337890625, 0.00833892822265625, -1.2705078125, -0.309326171875, -1.33984375, -1.150390625, 0.64990234375, 0.054534912109375, 0.826171875, -0.943359375, 0.626953125, -0.60595703125, -0.1463623046875, 1.109375, -1.310546875, 1.033203125, -1.9599609375, 1.33203125, -2.064453125, -0.0258636474609375, 0.6845703125, 0.1802978515625, -1.3779296875, -1.75, 1.3642578125, 1.7197265625, -2.126953125, 0.724609375, -0.272705078125, 1.16796875, -0.90234375, 0.67529296875, -1.5654296875, 1.3291015625, 0.681640625, 0.92626953125, 0.217529296875, -0.8515625, -0.84375, 0.29931640625, 0.662109375, -1.861328125, -0.4736328125, -1.849609375, -0.8955078125, 1.0048828125, -2.560546875, 0.87890625, 1.828125, -1.7333984375, 0.96142578125, 0.94384765625, 1.1015625, 0.67138671875, -0.6845703125, 1.3759765625, -1.1875, -2.365234375, 0.2174072265625, 1.0166015625, 0.56689453125, 0.38525390625, -0.480712890625, 1.62890625, 0.68798828125, -2.92578125, -0.7861328125, 1.1767578125, 1.009765625, 0.29296875, 0.0909423828125, 0.485595703125, 0.411376953125, -0.54296875, 0.436279296875, -0.232421875, -0.939453125, -0.82080078125, 0.374267578125, -0.491943359375, 1.5703125, -0.5087890625, 0.1468505859375, -1.955078125, 1.787109375, 0.73681640625, 0.2379150390625, -1.5947265625, 0.89404296875, 1.39453125, 0.2205810546875, 0.31494140625, -0.873046875, 0.468994140625, -0.13330078125, -0.1229248046875, -0.64794921875, -0.385498046875, -0.6884765625, 0.7138671875, 0.7998046875, 0.4453125, 1.634765625, 0.376708984375, 0.7724609375, 0.070068359375, -0.130615234375, 1.1328125, -1.5888671875, -0.466064453125, -1.2197265625, -0.53173828125, -0.39501953125, 0.1842041015625, -0.55126953125, -0.417724609375, -0.274658203125, -0.5888671875, 0.50048828125, 0.167724609375, 0.73486328125, 0.37890625, -0.301025390625, 0.294921875, 0.49462890625, -0.26513671875, -0.64111328125, 0.53662109375, -0.7099609375, -0.603515625, -0.681640625, 0.650390625, -0.0241546630859375, 0.44873046875, 0.9638671875, 1.7353515625, -1.8349609375, -0.1748046875, 1.9150390625, -0.52197265625, -0.2027587890625, 2.10546875, 1.345703125, 0.62890625, -0.63134765625, -0.6435546875, 0.1591796875, 1.0908203125, -0.619140625, -0.86865234375, 0.89208984375, 2.828125, -1.201171875, -0.318603515625, 0.96875, -0.0207061767578125, 1.0400390625, -0.86962890625, -1.0517578125, -0.489501953125, 1.994140625, 0.415283203125, -1.02734375, 1.5673828125, -0.99365234375, -0.0210113525390625, 0.79296875, -0.6953125, 0.65380859375, -1.15234375, 1.1650390625, 1.1875, 0.18115234375, 0.8583984375, -1.599609375, -0.87060546875, -0.63232421875, 1.84375, 0.875, -0.1783447265625, 0.98388671875, 1.5498046875, 1.4931640625, -0.04071044921875, -0.12451171875, 0.70166015625, -0.69091796875, 0.264404296875, 0.43603515625, -1.0341796875, 1.1494140625, 0.044769287109375, 1.6044921875, 1.279296875, 0.36328125, 3.01171875, -0.9912109375, -0.2379150390625, -1.1904296875, -1.37890625, -1.2421875, -1.0673828125, -1.2177734375, -1.392578125, 0.3623046875, -0.64013671875, -0.337158203125, 0.223876953125, -0.869140625, -0.1829833984375, 1.7890625, -0.135986328125, 0.6435546875, 0.4130859375, -0.96533203125, 0.7138671875, 0.7255859375, -1.4716796875, 0.7685546875, 0.423828125, 0.2294921875, -0.357177734375, 0.474609375, 2.07421875, 0.9052734375, 0.85498046875, 0.313720703125, -1.29296875, 1.9736328125, 0.49072265625, -1.2041015625, 0.6435546875, 0.03863525390625, 0.292236328125, 1.5185546875, -0.2568359375, 0.5302734375, -0.2744140625, -0.260009765625, 0.64990234375, 0.56982421875, 1.5029296875, 1.3154296875, 0.40478515625, -1.76171875, 0.06707763671875, 1.587890625, -2.099609375, 0.1453857421875, 0.52197265625, 2.279296875, 0.2281494140625, -0.402587890625, 1.39453125, 0.27685546875, -0.60546875, -1.6044921875, 0.32177734375, -0.2191162109375, -0.85498046875, -0.1832275390625, 1.1845703125, 1.017578125, -0.97705078125, 1.255859375, 1.1650390625, -0.9482421875, 1.501953125, 1.8896484375, -0.45361328125, 1.1796875, 1.10546875, -0.03643798828125, 0.3076171875, -2.240234375, 0.9326171875, -0.798828125, 0.0203399658203125, -0.351806640625, 1.234375, 1.361328125, 0.6533203125, 0.11016845703125, -1.369140625, 0.65380859375, -1.611328125, -0.958984375, 0.75732421875, -0.49267578125, 0.857421875, 0.5654296875, -0.73486328125, 0.07861328125, -0.83837890625, -1.2578125, 1.5126953125, 0.189697265625, 0.59912109375, 0.48291015625, -0.2171630859375, -0.007076263427734375, -0.66748046875, -0.572265625, -0.57080078125, 0.41796875, 0.404296875, 0.10540771484375, -0.6171875, 0.6201171875, 1.1455078125, -0.325439453125, -0.55126953125, 1.1103515625, -0.9287109375, -2.859375, 0.986328125, -0.9208984375, -0.99072265625, -0.3193359375, 0.87353515625, 0.54931640625, -1.1787109375, 0.2467041015625, 0.66015625, 1.931640625, 0.87158203125, 0.62890625, -0.0211639404296875, -0.038787841796875, 0.320068359375, 0.61962890625, 1.1171875, -0.73486328125, 0.267822265625, 0.1923828125, -1.482421875, 0.572265625, 1.8857421875, -1.83203125, -0.75927734375, -1.0986328125, 0.474609375, 1.1552734375, 0.6142578125, 0.41650390625, 0.9833984375, 0.359375, 0.1553955078125, -0.6572265625, 1.8203125, 0.47900390625, 1.7587890625, 1.31640625, 1.509765625, 0.94921875, 0.1998291015625, -1.8720703125, 1.1123046875, -0.2340087890625, -1.4208984375, 0.03045654296875, 0.92919921875, -0.87841796875, -0.1458740234375, 0.447265625, 0.95458984375, -0.546875, 0.3681640625, 0.90771484375, -1.529296875, 0.38134765625, -0.01715087890625, -0.08892822265625, -1.96875, 0.58642578125, -1.0009765625, -1.017578125, -2.109375, -0.61474609375, -2.328125, -0.900390625, -0.038177490234375, -0.0994873046875, 2.064453125, -0.6064453125, -0.0750732421875, -0.444091796875, -0.1492919921875, 1.177734375, 0.568359375, 2.4140625, -0.79150390625, 0.2919921875, -0.59375, 0.76953125, 0.615234375, -0.53955078125, -0.30224609375, -0.0271148681640625, -1.0556640625, 0.41845703125, 0.69970703125, 1.384765625, 0.67626953125, 0.5048828125, 0.73486328125, -0.6376953125, 1.2236328125, -0.1737060546875, 1.38671875, 0.8134765625, 0.485595703125, 0.07257080078125, 0.09344482421875, 0.092529296875, -0.01032257080078125, -1.3193359375, -0.7451171875, -0.2734375, -1.1513671875, -1.8134765625, -0.28271484375, -0.441162109375, 0.0421142578125, -0.085693359375, -0.7861328125, -0.186767578125, 0.72021484375, 0.3388671875, -0.447998046875, -0.8525390625, 1.79296875, 0.143798828125, -0.0013217926025390625, 0.8203125, -0.352294921875, 1.2509765625, 0.73779296875, 0.94189453125, -0.488525390625, -1.4951171875, -0.30126953125, 0.5947265625, -1.75390625, -0.4453125, -0.32568359375, -1.017578125, 0.2174072265625, 0.27685546875, 0.435791015625, -1.8115234375, -0.1805419921875, -0.395263671875, -0.09814453125, 1.5, -0.56787109375, -0.1114501953125, -1.2900390625, 0.5390625, 0.58203125, -0.72802734375, 0.1331787109375, -0.5732421875, -0.955078125, 1.4169921875, 0.445556640625, -1.1162109375, -1.6982421875, -1.15234375, 1.7373046875, -0.8994140625, 1.3701171875, -1.5966796875, 0.7392578125, 1.3798828125, -0.378173828125, 0.2802734375, -0.2403564453125, -0.27978515625, -0.6552734375, -0.67431640625, -0.38330078125, -1.193359375, -1.1318359375, 0.4443359375, 1.6376953125, 1.439453125, 0.60791015625, 0.278564453125, 1.4150390625, 0.36572265625, 0.96484375, 0.2393798828125, 0.04864501953125, 0.0360107421875, 0.314453125, 0.76171875, 0.389404296875, -0.168701171875, -0.06536865234375, -1.681640625, 1.0283203125, 0.1461181640625, -0.37841796875, 0.0994873046875, 0.274169921875, -0.33544921875, 2.171875, 0.73828125, 1.103515625, -1.53515625, -0.426513671875, -0.260498046875, 0.7568359375, -0.2724609375, -0.12237548828125, 0.12078857421875, -3.083984375, 0.830078125, -0.49853515625, -0.5009765625, 0.93017578125, 0.52392578125, 0.248291015625, 1.00390625, -1.658203125, 0.1202392578125, 0.1578369140625, -0.7333984375, 0.421875, 0.55615234375, -0.377197265625, -0.62939453125, -0.377197265625, 0.8017578125, -0.1519775390625, 0.7685546875, 0.0199737548828125, -0.200927734375, -0.18212890625, 1.546875, -0.53369140625, -1.62890625, 1.08984375, -1.3779296875, -0.1572265625, 0.287841796875, -0.57177734375, 0.11029052734375, 0.18310546875, 0.5146484375, 1.1416015625, -0.81982421875, -0.29833984375, -0.58837890625, -0.1009521484375, 0.45556640625, 0.3515625, 0.3671875, 0.95556640625, 0.434814453125, -0.1925048828125, -0.278076171875, -0.8251953125, -1.0263671875, 1.90234375, 1.294921875, -1.099609375, -0.44189453125, -1.091796875, 1.234375, -1.796875, -0.7421875, -0.0675048828125, -0.458984375, 1.3134765625, 0.04937744140625, 2.537109375, -0.01448822021484375, 0.60791015625, -0.354248046875, -0.39111328125, 1.42578125, 2.470703125, -0.491455078125, 0.382080078125, 0.44140625, 0.41015625, -0.011260986328125, -1.1005859375, 0.7734375, 0.2210693359375, -0.607421875, -1.5693359375, -0.1475830078125, 0.65380859375, -0.60107421875, -0.04449462890625, 0.57275390625, -0.4951171875, 0.95654296875, 0.72802734375, 1.15234375, 0.1588134765625, 0.6533203125, 2.22265625, 0.177490234375, 0.132568359375, 0.25048828125, -0.705078125, -0.040802001953125, -1.1181640625, 1.3076171875, -0.74951171875, 1.6396484375, 0.6689453125, 0.33349609375, -1.0869140625, 0.74658203125, 0.67138671875, 0.3583984375, -1.251953125, -0.8603515625, -0.39892578125, -0.8310546875, -0.7919921875, 0.255126953125, -0.59521484375, -0.334228515625, -1.900390625, 0.73193359375, -0.572265625, -0.89794921875, -0.73046875, -0.06439208984375, -0.67529296875, -0.36474609375, -0.2274169921875, 0.1221923828125, 0.5673828125, 0.3935546875, -1.734375, -1.19140625, -1.537109375, 0.041748046875, 1.0947265625, 1.0712890625, -0.59814453125, 0.149658203125, 1.185546875, 0.77880859375, 0.408935546875, -1.0283203125, -2.06640625, -0.64599609375, 0.953125, 0.84375, 1.345703125, 0.54638671875, 0.7998046875, -0.0306396484375, 0.1485595703125, 0.0098419189453125, 1.1591796875, -0.61474609375, -0.1434326171875, 1.8779296875, -0.86328125, -0.73095703125, 0.08380126953125, -0.187255859375, -0.279296875, -0.5009765625, 0.047637939453125, 1.2158203125, -1.748046875, 0.24853515625, 0.884765625, -0.83251953125, 0.166015625, 1.044921875, -0.7421875, -0.48193359375, -1.7666015625, -0.342041015625, 0.5361328125, -1.0859375, 0.154052734375, -0.425048828125, -1.939453125, 0.62646484375, 0.40185546875, -1.65234375, -0.3134765625, -0.1837158203125, 0.043243408203125, -0.494140625, 0.41650390625, 0.630859375, 0.456787109375, -0.349365234375, -1.09375, -0.260498046875, -0.126220703125, -0.201416015625, 0.11358642578125, 0.10552978515625, 0.1380615234375, 0.91064453125, -0.79296875, -1.4599609375, 1.5546875, 0.52880859375, 0.264404296875, -1.0078125, 1.1865234375, -1.65625, 0.408447265625, 0.2763671875, 1.8740234375, 1.3291015625, 0.07415771484375, 0.357666015625, 0.61865234375, -1.068359375, 0.01043701171875, 0.1531982421875, -0.97998046875, 0.38818359375, -0.8828125, 1.189453125, -0.485107421875, 0.6806640625, 1.48046875, 0.474609375, 2.7578125, 1.142578125, 0.11541748046875, 0.472900390625, -1.0595703125, -0.50634765625, -0.397705078125, 0.82861328125, 0.32177734375, -1.0341796875, 1.09765625, -2.560546875, -0.62646484375, -1.0458984375, 0.35498046875, -2.96875, -0.135498046875, -0.4501953125, 1.0546875, 0.1566162109375, 0.49560546875, -0.351806640625, -1.162109375, 1.2333984375, -0.27099609375, -0.257568359375, 0.2137451171875, -0.1700439453125, 0.564453125, 0.244384765625, 0.97119140625, 0.2418212890625, 0.272705078125, 0.457275390625, -0.2193603515625, -0.0494384765625, 1.8173828125, 1.0400390625, 1.390625, 0.326416015625, 1.125, 0.158935546875, 0.43994140625, -1.201171875, -1.0751953125, -0.466552734375, -0.64404296875, -0.50146484375, -0.70654296875, 0.01264190673828125, 0.364990234375, 1.705078125, 0.4794921875, 0.6298828125, 0.98095703125, -0.861328125, -0.33251953125, -0.7177734375, -0.1331787109375, 0.45556640625, -0.857421875, -0.75341796875, 0.0228729248046875, -0.72900390625, -0.35888671875, 0.62109375, -0.2548828125, 0.70458984375, -0.0256195068359375, 0.56640625, -0.338134765625, -0.66162109375, -1.6796875, 0.3505859375, 0.050872802734375, 0.30859375, -3.017578125, 1.3896484375, 1.5830078125, 0.185546875, -0.6015625, 1.3349609375, -1.37109375, -1.3427734375, 0.853515625, 0.44580078125, 0.44091796875, -1.0830078125, 1.1162109375, -0.71826171875, -1.466796875, -0.445068359375, -0.428955078125, -0.3525390625, 0.094482421875, -0.041595458984375, -0.3837890625, 1.4345703125, -3.103515625, 0.931640625, -1.5830078125, 1.341796875, 0.377685546875, -0.0887451171875, 0.0189666748046875, -0.74365234375, 0.66064453125, 0.1773681640625, 1.6025390625, -0.4521484375, 0.1763916015625, 1.16015625, 0.10321044921875, 0.59033203125, 0.3095703125, -1.9853515625, 0.301513671875, -1.056640625, -0.28515625, 0.767578125, -0.88916015625, -0.69873046875, -0.1412353515625, -0.8037109375, 1.3134765625, 1.0908203125, -1.57421875, 0.057281494140625, -0.40576171875, 2.39453125, 0.39453125, -0.734375, 0.1641845703125, -0.451904296875, 0.2158203125, 2.712890625, -0.70068359375, 0.255859375, -0.83935546875, -0.213623046875, -0.302978515625, -0.280517578125, -0.27197265625, -0.422607421875, -1.294921875, 1.5498046875, 0.171142578125, -1.2099609375, 0.3173828125, 0.444580078125, -0.41650390625, -1.564453125, -0.9462890625, -1.2451171875, 0.5966796875, -0.33740234375, 1.81640625, -1.1083984375, -0.455078125, -0.22900390625, 0.32080078125, 0.251708984375, -0.12353515625, 0.0103302001953125, -1.365234375, -0.90478515625, 1.48046875, 0.38525390625, -1.337890625, -0.282470703125, -0.35498046875, 1.234375, 0.206298828125, -1.1884765625, -1.7041015625, 0.8681640625, 2.1484375, 0.045166015625, -0.78857421875, 0.326416015625, 0.2685546875, 0.200439453125, 2.193359375, -0.14794921875, 0.90283203125, -1.0830078125, -1.3056640625, 0.353515625, 1.3310546875, 1.7109375, -0.17041015625, 1.82421875, -0.00826263427734375, -0.1600341796875, -0.84814453125, 0.72900390625, -2.353515625, 1.0263671875, 0.63916015625, -1.4951171875, -0.427978515625, -0.84423828125, -0.269287109375, 0.418212890625, -1.0283203125, 1.833984375, -1.759765625, 0.66162109375, 0.08502197265625, -0.77490234375, 1.224609375, -0.20556640625, -0.5888671875, -1.9892578125, 0.55078125, -1.244140625, -0.30126953125, 1.3447265625, -0.4765625, 1.1689453125, 0.6640625, 0.388671875, 0.00963592529296875, -1.4345703125, 0.66162109375, -0.6845703125, 1.59765625, -1.056640625, -0.6875, 0.131591796875, -0.619140625, 0.658203125, -0.29638671875, -1.0908203125, 1.0751953125, 0.7744140625, -0.081298828125, -0.2005615234375, -0.6416015625, 0.63720703125, 1.525390625, 0.03350830078125, 1.08984375, -2.810546875, 1.2060546875, -0.73486328125, -1.2978515625, -0.2294921875, -1.6533203125, -1.0107421875, -0.8056640625, -0.040283203125, -0.81689453125, -1.5419921875, -0.94970703125, 0.923828125, -0.273681640625, 0.0870361328125, 0.1492919921875, -0.2249755859375, -0.412353515625, -1.359375, 0.90673828125, -0.5146484375, -0.71044921875, -0.923828125, -0.9052734375, -0.34033203125, 0.310546875, 0.9658203125, -0.451171875, -0.1268310546875, -0.455322265625, 0.6533203125, -0.32177734375, -0.53125, -0.21142578125, -1.22265625, -0.360107421875, -0.2227783203125, -0.433349609375, -0.83935546875, -0.8310546875, 0.7109375, 0.93408203125, 0.169677734375, -0.48828125, -0.26708984375, 0.1683349609375, -2.49609375, 0.5625, 0.1351318359375, 0.85107421875, 0.1849365234375, -1.9599609375, 0.61279296875, -0.349853515625, 1.4443359375, -0.41650390625, 1.2314453125, -0.09820556640625, -0.8603515625, -0.310302734375, -1.46875, 1.2890625, -0.047607421875, 1.56640625, 0.763671875, 1.234375, 0.61767578125, -0.271484375, -0.90869140625, -0.392333984375, -1.4541015625, 1.869140625, -0.016998291015625, -1.7783203125, -0.264892578125, -0.970703125, -1.0166015625, -0.338134765625, 0.91357421875, 0.78271484375, -0.18359375, -0.40087890625, -1.1376953125, -2.0703125, -0.351318359375, -0.66748046875, -1.4921875, -0.833984375, 0.38232421875, -0.97900390625, 0.025238037109375, 0.97509765625, -1.13671875, -0.262451171875, -0.0667724609375, -0.316650390625, 0.66259765625, 0.1759033203125, 0.86865234375, 1.150390625, 0.74462890625, -1.283203125, 0.050689697265625, 0.97900390625, 0.16162109375, 0.7392578125, 1.7314453125, 0.98828125, 0.83544921875, -0.8662109375, 0.896484375, 0.7900390625, -0.82275390625, -0.0247802734375, 1.1474609375, -1.1337890625, 1.2236328125, 1.845703125, -0.5849609375, 2.322265625, -0.1043701171875, -0.2052001953125, -0.87548828125, 0.1895751953125, 0.0814208984375, 1.0341796875, -1.2978515625, -1.1953125, 0.5166015625, 0.96435546875, 1.5029296875, -0.1805419921875, -0.96240234375, 0.391845703125, 0.478515625, 0.86279296875, 1.095703125, -0.296630859375, -0.72265625, -2.0390625, -0.6533203125, 0.371337890625, -0.76708984375, 0.067138671875, 0.6669921875, 1.611328125, 0.74755859375, 0.892578125, 0.68896484375, 0.07989501953125, -0.3212890625, 0.137451171875, -1.119140625, -0.8798828125, -0.401123046875, 1.0625, 0.11236572265625, 1.666015625, 1.7548828125, -1.08203125, 0.406982421875, -2.09375, -0.1976318359375, 0.4814453125, 0.4560546875, -0.43212890625, -2.0390625, -1.44140625, -0.0582275390625, -1.19140625, 0.7109375, -0.347900390625, 0.66064453125, 0.31884765625, 1.2509765625, 0.1552734375, 2.32421875, 0.689453125, -1.1181640625, 1.0947265625, -0.17626953125, 0.479248046875, 1.806640625, -0.005718231201171875, -0.34619140625, -0.65185546875, 0.69873046875, 0.728515625, 0.6123046875, -0.06243896484375, -0.79150390625, 1.02734375, -1.0478515625, 0.39208984375, -0.023529052734375, -1.5712890625, 1.0146484375, -0.98095703125, -0.72314453125, -0.31494140625, -1.6650390625, -1.091796875, -1.1064453125, 1.0712890625, -1.62109375, 0.1478271484375, 1.5615234375, 1.982421875, 2.279296875, -0.9794921875, 0.096435546875, 0.450927734375, -1.251953125, -2.703125, -0.059295654296875, 1.7412109375, 0.34228515625, 1.0849609375, 0.1951904296875, 1.09765625, 0.385009765625, -0.5341796875, 1.0498046875, 0.67919921875, 0.65869140625, 0.63037109375, 0.845703125, -1.2744140625, -0.82763671875, -0.826171875, 0.58154296875, -0.94140625, -1.02734375, 0.264404296875, 0.326904296875, 0.55908203125, -1.75, -0.1392822265625, 1.6962890625, 0.99560546875, -1.2861328125, -0.263916015625, -1.169921875, 0.6142578125, -0.47607421875, 0.00421142578125, -0.9580078125, -0.87109375, -0.45751953125, 0.3662109375, 1.8466796875, 1.77734375, 0.29296875, 1.46484375, -0.63427734375, 0.91845703125, -1.03515625, -0.708984375, 1.25, 0.6455078125, -1.830078125, -1.189453125, -1.123046875, -0.387451171875, 0.720703125, 1.8681640625, 0.28173828125, -0.60302734375, -0.391357421875, 0.303466796875, -1.333984375, -1.8671875, 1.3896484375, 0.7861328125, 0.1507568359375, -0.79052734375, 0.8955078125, -0.4365234375, -0.41455078125, 0.2271728515625, 1.5771484375, 1.1328125, 0.703125, 0.118896484375, 0.08538818359375, 1.875, -0.60595703125, -0.1468505859375, -0.454833984375, 1.5390625, -0.751953125, -1.345703125, 0.2088623046875, 0.310546875, -0.2430419921875, 0.341064453125, 0.654296875, 0.47607421875, -0.45751953125, -0.17333984375, -0.4482421875, -0.6748046875, -0.61962890625, 0.400146484375, -0.276611328125, -2.134765625, -0.28125, 0.62890625, -1.6435546875, 0.66455078125, -1.1572265625, -0.404052734375, -0.64306640625, -0.469970703125, 0.40771484375, -1.173828125, 1.2421875, -0.7412109375, 0.03607177734375, 0.14208984375, 0.1768798828125, 1.3212890625, 0.95458984375, -0.74853515625, -0.9501953125, 0.0723876953125, 0.77490234375, 0.1412353515625, -1.3212890625, -0.08575439453125, -0.48046875, -0.1572265625, -1.462890625, -0.9814453125, -0.040740966796875, 1.439453125, -0.64990234375, -0.08477783203125, -0.33349609375, -1.1611328125, -0.8798828125, 0.767578125, -1.0859375, -0.337890625, -0.857421875, 0.6435546875, 0.96044921875, -0.7451171875, 1.4228515625, -0.1759033203125, 0.03326416015625, -0.167724609375, 0.63720703125, -1.0234375, 0.024658203125, 0.77392578125, 0.4873046875, -1.4541015625, -1.5537109375, -0.79638671875, -1.916015625, -1.06640625, -0.90771484375, -0.26416015625, -0.59716796875, -0.8076171875, 1.12109375, 0.75537109375, 0.5537109375, -0.6572265625, -1.1845703125, -0.94580078125, 0.27197265625, 0.67822265625, 0.84033203125, 0.8173828125, 0.445556640625, -0.72802734375, 0.2362060546875, -0.6181640625, -0.126220703125, 0.08038330078125, 0.5625, 2.009765625, 1.6533203125, -0.72119140625, 0.7724609375, 0.06842041015625, -0.71826171875, 0.423095703125, 2.28125, -0.222900390625, -0.389892578125, -0.1455078125, 0.09173583984375, 0.07745361328125, -2.123046875, -1.1220703125, 0.564453125, -1.3291015625, 0.53125, 0.433837890625, 0.5888671875, -0.207275390625, -0.59130859375, 0.50439453125, 0.97509765625, 2.59765625, 2.765625, -0.1263427734375, -0.155517578125, -0.55078125, -1.1806640625, 1.1279296875, 0.3857421875, 0.348876953125, 1.8427734375, 0.425048828125, 0.488037109375, -0.25390625, 0.97412109375, -0.423583984375, -1.05859375, -0.452880859375, 1.017578125, 0.89111328125, 2.041015625, -1.73046875, -0.1947021484375, 0.41748046875, 1.9658203125, 0.01175689697265625, -2.93359375, -0.357177734375, 1.430511474609375e-06, 1.0224609375, -0.59130859375, 0.24462890625, -0.099609375, -1.404296875, 0.83935546875, -1.240234375, 0.7705078125, -1.5126953125, 1.5, -1.5302734375, 1.1123046875, -0.45751953125, 1.8603515625, -0.10205078125, -0.8173828125, -2.119140625, 0.44873046875, 0.08807373046875, 0.380859375, -0.69775390625, -2.099609375, 1.130859375, -0.5615234375, 2.087890625, 2.08203125, 0.189697265625, 0.2432861328125, 0.0027008056640625, 1.666015625, 0.54833984375, 0.99755859375, -1.62109375, -1.1923828125, -0.342529296875, -0.9599609375, -0.57568359375, 1.5810546875, 0.1407470703125, -0.0606689453125, -0.1326904296875, 0.71142578125, 0.67578125, 0.65478515625, 0.0751953125, -1.138671875, 0.08819580078125, 0.4599609375, -0.371337890625, 0.841796875, 0.615234375, -0.53076171875, 1.4609375, 0.43115234375, 1.0322265625, 0.681640625, 0.53173828125, 0.93798828125, 1.2314453125, -0.4970703125, -0.775390625, -2.3203125, 0.78271484375, -1.2529296875, 1.197265625, -0.320068359375, 0.568359375, -0.283447265625, 0.084228515625, -1.8740234375, 0.818359375, -2.5234375, -0.53173828125, -0.0833740234375, -1.142578125, -0.0438232421875, 1.4404296875, -0.52490234375, -0.2283935546875, 0.266357421875, -0.7783203125, -0.42919921875, 0.07879638671875, -0.5146484375, -0.08624267578125, 0.2064208984375, -0.64208984375, -0.595703125, 1.3466796875, 0.076904296875, 1.6357421875, -0.2117919921875, -0.053741455078125, -0.77978515625, 0.3427734375, 0.1436767578125, -0.10711669921875, -0.2178955078125, -1.080078125, 0.05267333984375, 1.3330078125, -0.873046875, 1.130859375, -1.7294921875, -0.461181640625, -0.7900390625, 0.070556640625, -1.62109375, 0.1761474609375, 0.31591796875, 0.48046875, 0.63525390625, 0.90966796875, -0.80712890625, -2.357421875, -0.55517578125, -1.609375, -0.90771484375, -1.3349609375, 0.2021484375, 0.1846923828125, 1.06640625, 2.76953125, 0.57958984375, -0.484619140625, 0.59326171875, -0.9580078125, -0.8974609375, -0.3515625, 0.1455078125, -0.11920166015625, 0.78173828125, -2.578125, -1.1123046875, 0.77099609375, 0.390625, -0.63623046875, 0.376708984375, 0.1949462890625, 1.76953125, 0.0125885009765625, 2.591796875, -2.353515625, 0.515625, 0.48681640625, -0.67431640625, 0.181396484375, 0.51220703125, 0.337646484375, -0.1295166015625, -0.513671875, 1.083984375, -0.057891845703125, -0.044219970703125, -2.28515625, 0.953125, -0.96728515625, 0.84716796875, -0.1558837890625, 0.050048828125, -0.720703125, -1.1142578125, -1.634765625, 1.7724609375, 0.0289154052734375, -1.810546875, 1.2119140625, 0.0916748046875, 0.419189453125, -1.158203125, -1.41015625, -0.4501953125, -0.09344482421875, 0.60107421875, -0.88232421875, 1.8701171875, -1.5126953125, -1.4716796875, 1.6455078125, 0.31298828125, -0.02496337890625, 0.264404296875, 0.1468505859375, -0.378662109375, -1.0732421875, 1.677734375, -0.409912109375, 0.21630859375, -0.583984375, 0.09161376953125, 2.04296875, -0.81298828125, -1.16796875, 1.193359375, 0.52978515625, -1.3017578125, -0.59814453125, 0.8583984375, 0.09002685546875, -0.8134765625, 1.1435546875, -0.96142578125, 0.53564453125, -0.96435546875, -1.8310546875, 0.48828125, 0.202880859375, 0.369140625, 0.51416015625, -1.0244140625, -0.398681640625, 0.55029296875, -0.450927734375, 0.0185699462890625, -0.81494140625, -0.0089263916015625, -0.91650390625, -1.349609375, -0.329833984375, -0.10870361328125, -2.314453125, 0.358642578125, -1.7001953125, 0.84619140625, 0.66162109375, -0.061767578125, 0.64599609375, -0.01013946533203125, -0.55419921875, -0.39111328125, -0.89404296875, 0.931640625, 1.896484375, 1.373046875, 0.1553955078125, -0.1856689453125, -0.1959228515625, -1.712890625, 0.261962890625, 0.364501953125, -1.708984375, 0.67919921875, 0.96923828125, 0.097900390625, 0.21875, 0.8232421875, -0.68505859375, 0.75439453125, 0.8525390625, 1.615234375, -1.1875, 1.0380859375, -0.529296875, -0.51220703125, 0.32861328125, -2.03125, -0.19384765625, -1.1044921875, 0.7236328125, 0.15283203125, -0.7607421875, -1.4130859375, -0.9140625, 0.09039306640625, 0.2110595703125, -0.63623046875, -0.55126953125, -0.564453125, -1.322265625, -0.2244873046875, 0.791015625, 0.35595703125, 0.658203125, -0.8349609375, -1.248046875, -0.425537109375, 0.21923828125, 0.477294921875, 0.990234375, 0.99365234375, 0.08343505859375, -2.052734375, -0.6708984375, 0.5810546875, 0.69189453125, -0.07635498046875, -0.07183837890625, 1.7314453125, -0.297607421875, 1.0810546875, 0.7978515625, -0.39892578125, -0.2213134765625, -0.1353759765625, 0.03717041015625, 1.3447265625, -0.09600830078125, -0.354736328125, 0.5634765625, -0.1256103515625, -0.26953125, 1.486328125, -0.5390625, -0.7431640625, -0.251708984375, -1.5625, -0.404296875, 0.8251953125, 0.31689453125, -1.564453125, 0.66015625, -1.201171875, -1.15625, 0.262451171875, -0.71923828125, 0.38525390625, -0.385986328125, -2.931640625, 1.3134765625, 0.5390625, 0.43408203125, 0.25341796875, -0.12396240234375, 0.5126953125, -0.2880859375, -2.357421875, 0.017120361328125, -0.1568603515625, 0.01296234130859375, 0.7666015625, -0.339111328125, -0.247802734375, -1.3427734375, -0.84765625, 0.8759765625, 0.97119140625, -0.10467529296875, -0.37255859375, 0.2216796875, 0.059967041015625, -0.26513671875, -0.290771484375, -0.95068359375, 0.442626953125, -0.68310546875, -0.68310546875, 0.9873046875, -0.734375, 1.8828125, 0.12322998046875, -1.126953125, -0.08880615234375, 0.60888671875, -0.07208251953125, -1.38671875, -0.7724609375, -0.75927734375, 0.97900390625, -0.220458984375, -0.01190948486328125, 0.00968170166015625, 1.2197265625, -2.11328125, 0.30712890625, 0.66845703125, -0.040130615234375, 1.5947265625, 1.455078125, -0.85693359375, -0.91455078125, -1.5322265625, 0.023040771484375, 0.5, -0.5693359375, 0.9033203125, -0.83349609375, 0.97802734375, -0.82763671875, 1.677734375, -1.2041015625, 0.0709228515625, -0.399169921875, -1.8212890625, -2.771484375, 1.4228515625, -0.365234375, -0.90087890625, -1.642578125, 0.48876953125, 0.6572265625, -0.366455078125, -0.434814453125, 0.67431640625, 1.2451171875, 0.313720703125, -0.1998291015625, -0.32373046875, -0.70361328125, 1.65625, 2.38671875, -0.7919921875, 0.77978515625, 0.058258056640625, -0.70751953125, -0.0012464523315429688, 1.111328125, -0.95458984375, -0.64501953125, -0.33056640625, 0.0458984375, 0.357666015625, 0.53857421875, 1.2578125, -1.1494140625, 0.1234130859375, -0.6337890625, 1.5107421875, 1.1611328125, 1.517578125, 0.2281494140625, -0.2265625, 0.188232421875, 1.1494140625, -0.0157928466796875, 1.0, -1.4619140625, -0.290771484375, -0.537109375, -0.55615234375, 0.09503173828125, -0.114013671875, 0.0906982421875, -1.5185546875, 1.2587890625, -0.9541015625, 0.599609375, -1.2841796875, -0.3779296875, 1.5712890625, 1.173828125, -1.044921875, -0.13525390625, -0.051605224609375, 1.974609375, -0.57080078125, 0.7109375, -1.2158203125, -0.280029296875, -0.98095703125, 0.31494140625, 0.311767578125, 1.046875, 0.77587890625, 0.93798828125, 0.452880859375, 0.277587890625, -0.958984375, 0.378173828125, -0.1270751953125, -0.268310546875, 0.443115234375, 1.6826171875, -0.48486328125, -0.0523681640625, 0.181884765625, -0.91552734375, 1.3857421875, -0.011627197265625, 0.2021484375, -0.61376953125, -0.7978515625, 0.379150390625, 0.96923828125, 0.60986328125, 0.1702880859375, -0.6640625, 0.93017578125, 0.6669921875, 0.274169921875, -0.343505859375, -0.85693359375, -0.5673828125, -1.5146484375, -0.0203399658203125, -0.29296875, 0.239501953125, 0.71630859375, -0.225341796875, -0.343505859375, 0.72705078125, -1.4462890625, 1.041015625, 1.2841796875, 1.4775390625, -1.07421875, 0.00811767578125, -0.1434326171875, 0.49169921875, -0.164306640625, -0.64697265625, 0.061004638671875, -0.66845703125, 0.068115234375, -1.380859375, -0.5322265625, 0.72900390625, -0.142578125, 0.68896484375, -2.6015625, 1.130859375, 0.328369140625, -0.2215576171875, -0.5966796875, 0.449951171875, -1.9150390625, 0.208984375, -0.02813720703125, 0.6865234375, 0.94580078125, -0.1763916015625, 0.11602783203125, -0.0604248046875, 0.97509765625, -0.375732421875, -0.4892578125, 1.0537109375, -1.8671875, 1.900390625, 1.0322265625, 0.6162109375, -0.71875, 0.1202392578125, 1.248046875, -2.201171875, -0.46630859375, -0.84326171875, 0.159912109375, -0.99072265625, 1.0849609375, 1.36328125, 1.7958984375, 0.1337890625, -2.18359375, 0.83935546875, 0.17236328125, 1.01171875, 1.4501953125, -0.7578125, 1.5087890625, -0.1234130859375, -0.01251983642578125, -0.85205078125, 0.432861328125, 0.07489013671875, 0.30419921875, -0.5703125, 1.5771484375, 1.6005859375, -1.263671875, 0.59326171875, -0.395263671875, 0.22998046875, -0.2274169921875, -0.0751953125, -0.0271453857421875, -0.267822265625, -0.3330078125, 1.3447265625, 0.57470703125, -1.64453125, -1.51953125, 0.434326171875, 0.41943359375, -1.1005859375, -0.7509765625, -1.248046875, -0.18310546875, 0.9130859375, -1.8544921875, -0.51123046875, 1.1904296875, -0.1085205078125, -0.495361328125, -0.658203125, -0.2880859375, 0.2161865234375, 1.6044921875, 0.2408447265625, 0.6474609375, 1.0234375, -0.9130859375, 1.0751953125, 0.005462646484375, -0.1258544921875, -1.125, -0.79150390625, -1.779296875, 1.390625, 0.0241546630859375, -2.544921875, -1.0478515625, 1.796875, -1.712890625, -1.66796875, -0.6337890625, -0.8466796875, -1.1982421875, -0.2215576171875, -1.2412109375, -1.4951171875, -0.6416015625, -1.83203125, 0.01171875, -0.43994140625, -1.419921875, -1.228515625, -0.98388671875, -0.53955078125, 2.6640625, -0.477783203125, 0.66064453125, -0.52978515625, 1.1064453125, -0.8173828125, 0.8525390625, -0.328125, 1.4833984375, 1.01953125, 0.3447265625, -0.5625, 0.2188720703125, -1.9423828125, 0.310302734375, -0.202880859375, -0.99658203125, 0.560546875, 0.56689453125, 0.51220703125, -1.2763671875, 0.92138671875, 0.218994140625, 0.054718017578125, 0.427001953125, -1.5625, -0.34765625, -1.587890625, 3.310546875, -1.591796875, 1.607421875, -0.4560546875, -0.7431640625, 0.265625, 0.27880859375, -0.54541015625, 1.01953125, 0.81494140625, -0.6611328125, -0.830078125, -1.0498046875, 1.3369140625, -1.45703125, -1.16015625, 1.4267578125, -0.228515625, 1.65234375, 0.0017175674438476562, -0.181396484375, -0.53515625, -0.63720703125, -0.63525390625, -0.368896484375, -0.1334228515625, 2.017578125, 2.041015625, -0.11053466796875, -1.2802734375, -0.84814453125, -0.254638671875, -1.5966796875, -0.67431640625, -0.80126953125, -0.361328125, 2.62890625, 0.486083984375, 0.08636474609375, 0.6162109375, -0.8046875, -0.5078125, 0.97265625, 0.386474609375, 1.248046875, -0.0120391845703125, -0.92626953125, 0.382080078125, 0.533203125, 0.69775390625, -0.9462890625, -1.201171875, 1.4677734375, -0.140380859375, -1.134765625, -1.69140625, 0.56787109375, -0.59326171875, -0.01036834716796875, -0.97607421875, -2.240234375, 0.60498046875, 0.25048828125, 0.72216796875, -0.312255859375, -0.343017578125, -0.83642578125, -0.455810546875, -0.2401123046875, -0.68701171875, 0.1439208984375, 0.11199951171875, -0.390869140625, -0.70458984375, -1.07421875, -0.8515625, 0.283203125, 0.08502197265625, -0.7021484375, -0.06451416015625, -0.2283935546875, -0.1719970703125, -0.082763671875, 0.78662109375, 0.50439453125, 0.364990234375, -0.37646484375, -1.158203125, 0.896484375, -2.865234375, 0.244873046875, 0.406005859375, -0.1533203125, 0.032318115234375, -0.006134033203125, 1.1484375, -0.92236328125, 0.09515380859375, -0.99951171875, 0.8544921875, -1.3369140625, 0.8466796875, -0.59912109375, 0.5693359375, -2.443359375, 1.603515625, -0.403076171875, -0.556640625, 0.85107421875, -0.3271484375, 0.7265625, -0.6796875, -1.7373046875, 1.1533203125, -0.297607421875, 0.262939453125, 0.01334381103515625, -0.0650634765625, -1.8623046875, 0.78076171875, 1.72265625, -0.5458984375, 0.43408203125, -0.03851318359375, 0.49853515625, -0.2154541015625, -0.32470703125, 0.055145263671875, -0.46728515625, 0.615234375, 0.44091796875, -0.37744140625, 0.85302734375, 0.053680419921875, 0.6640625, -1.7001953125, -0.77880859375, 0.6318359375, -0.95263671875, -0.0989990234375, 0.671875, -0.13232421875, -0.97314453125, -0.460205078125, -0.541015625, -0.39990234375, 0.1773681640625, 0.13037109375, -0.68994140625, 0.58544921875, 0.2069091796875, -0.5126953125, -1.1005859375, -2.94140625, -0.390380859375, -1.5537109375, 1.4951171875, -0.67333984375, 0.3408203125, -1.412109375, 1.568359375, 2.232421875, -0.662109375, -0.81884765625, -0.0462646484375, 0.1387939453125, -0.92138671875, -0.305419921875, 0.151123046875, -0.787109375, 0.489990234375, -1.3984375, 1.279296875, 0.91552734375, 0.6259765625, -0.60107421875, -0.319580078125, -0.76220703125, -0.319091796875, -0.7900390625, 0.83642578125, 0.2001953125, -1.5087890625, 0.044830322265625, -0.8564453125, -0.142333984375, -0.385986328125, -1.466796875, -0.97607421875, 0.46533203125, -0.268798828125, -0.141357421875, 1.078125, -1.9443359375, 0.0235137939453125, 0.0188751220703125, 1.6044921875, -0.1669921875, 1.3173828125, 0.5830078125, 0.4716796875, -0.28125, 0.552734375, -2.478515625, 0.6728515625, 0.75390625, 0.50439453125, -1.291015625, -0.5703125, -0.482177734375, -0.16064453125, 0.09979248046875, 0.458984375, 0.37109375, -0.41259765625, -0.60107421875, -0.703125, -0.62548828125, 0.8662109375, 0.87255859375, 0.385009765625, 0.0177764892578125, -1.265625, 1.12890625, -0.466796875, 0.5830078125, 0.414794921875, -0.97802734375, 0.91015625, 0.314453125, -0.5869140625, -0.77734375, -0.37646484375, -1.17578125, -0.853515625, -0.40771484375, 0.56396484375, 0.08837890625, 0.204345703125, 0.5380859375, -1.2900390625, -0.234130859375, -0.1341552734375, -0.039886474609375, 2.119140625, 0.9677734375, -1.27734375, 0.0870361328125, -1.021484375, -0.9541015625, 0.78955078125, 2.912109375, -0.18017578125, 0.25244140625, 0.24755859375, 0.92724609375, -1.58203125, 1.68359375, -0.82421875, -1.123046875, -0.048370361328125, -0.0672607421875, 0.1285400390625, -0.73779296875, 0.55712890625, 0.24658203125, -0.8662109375, 0.31396484375, 0.011077880859375, 0.386962890625, -0.2076416015625, 0.06646728515625, -0.2568359375, 0.85986328125, -0.0703125, 1.1474609375, -0.92578125, 0.59375, -1.083984375, -0.49853515625, 1.5126953125, -0.385986328125, 1.533203125, 0.5927734375, 1.1826171875, 0.47509765625, 1.5537109375, -0.49951171875, -0.153076171875, -0.578125, -1.1201171875, 0.693359375, 0.90869140625, 0.26123046875, -0.106689453125, -0.07574462890625, -0.1741943359375, 2.05078125, -0.0141143798828125, 0.54345703125, -2.177734375, 0.9931640625, 0.09759521484375, -0.0789794921875, -0.697265625, -1.2158203125, -0.8466796875, -0.3349609375, 1.2626953125, -0.439208984375, -3.26953125, 1.3681640625, -2.49609375, -1.41796875, -1.6083984375, -0.7744140625, -1.2734375, 0.98828125, -0.71240234375, 1.1298828125, -0.8857421875, 1.2548828125, -0.146484375, -1.0517578125, 0.4013671875, -0.0540771484375, -0.0555419921875, 1.193359375, -2.23828125, -0.258544921875, 1.744140625, 0.114990234375, 0.03851318359375, 0.884765625, 0.335693359375, -0.206298828125, -0.2476806640625, 0.9755859375, -0.0010137557983398438, 1.6298828125, 0.14599609375, 0.177734375, 0.03558349609375, 0.031463623046875, 1.0478515625, -0.56787109375, -0.129638671875, 0.1702880859375, -0.157470703125, 1.474609375, 0.57080078125, -1.154296875, -0.44140625, 0.136474609375, 0.583984375, -0.1524658203125, 0.4609375, 0.288818359375, 0.6123046875, 0.13623046875, 0.6474609375, -1.19921875, 1.265625, -0.888671875, -0.1376953125, 0.6201171875, -0.001590728759765625, -0.23779296875, -0.24267578125, 0.05889892578125, -0.52587890625, -1.65625, -2.787109375, -0.93896484375, -0.438232421875, -0.482421875, 1.501953125, 0.71826171875, 0.62353515625, -0.0015687942504882812, 1.1904296875, -1.7841796875, 0.444580078125, -0.50634765625, -0.1756591796875, -0.8232421875, -0.0216522216796875, 1.0341796875, -2.13671875, 0.2626953125, -0.55029296875, 0.25, 0.261474609375, 0.720703125, -0.76904296875, 0.225830078125, 0.640625, 1.5751953125, 0.85009765625, -0.59521484375, -1.3837890625, 0.98046875, 0.050201416015625, -0.1407470703125, -0.89453125, 0.892578125, -0.27001953125, -1.38671875, -0.8330078125, 1.4228515625, 0.446533203125, -1.39453125, -0.53125, 0.2235107421875, -0.44287109375, -0.685546875, 0.57421875, -0.80712890625, -0.38427734375, 0.07379150390625, 0.348876953125, -1.0693359375, -0.477783203125, -0.9931640625, -0.195068359375, 1.4697265625, 0.332763671875, 1.232421875, 1.1484375, 2.376953125, -0.0308074951171875, -0.328369140625, -0.362548828125, 0.419921875, 0.572265625, -1.0400390625, -0.5439453125, 1.6015625, -0.4658203125, 0.332275390625, 1.029296875, -0.578125, 0.7021484375, -1.4404296875, 0.42724609375, 1.1318359375, -1.1162109375, 0.36279296875, 1.0380859375, -1.005859375, 0.693359375, 0.1192626953125, -0.214111328125, 1.392578125, 0.841796875, -0.233642578125, 0.52685546875, -0.8349609375, 0.19775390625, -0.69873046875, 0.64453125, 0.10272216796875, -0.951171875, -0.90576171875, 0.1395263671875, -0.30419921875, -1.0146484375, -0.81787109375, -1.5068359375, 0.86181640625, 1.9296875, -1.080078125, -1.66796875, -0.291259765625, -0.53173828125, 0.265869140625, 0.60498046875, -0.1103515625, 0.74755859375, -0.88916015625, 1.91015625, -0.1884765625, 0.57373046875, -0.08203125, -0.2425537109375, 1.306640625, -0.7314453125, 0.12744140625, 0.58349609375, 0.80078125, 1.11328125, -0.32080078125, -0.41357421875, 1.0185546875, 1.01953125, -0.53466796875, -0.50537109375, -0.07244873046875, 0.0955810546875, -0.5693359375, -0.51904296875, -0.29541015625, 1.0234375, -0.86376953125, -0.454345703125, 0.904296875, 0.487548828125, 0.97412109375, 0.6064453125, 1.486328125, -0.61865234375, -1.171875, 0.09564208984375, -0.95751953125, 0.05670166015625, 0.2822265625, -0.25341796875, 0.26611328125, 1.4990234375, 0.262939453125, 2.12890625, -0.6796875, -1.5732421875, -0.151611328125, 0.397705078125, 0.71630859375, -0.61669921875, -0.2490234375, -0.8984375, -0.497314453125, -0.102294921875, -0.092529296875, 1.0732421875, -0.81884765625, -0.2890625, -0.306640625, -0.6982421875, 0.055419921875, -0.37353515625, 1.833984375, 0.12060546875, -0.47265625, 0.480224609375, 1.5009765625, -1.6142578125, -0.1968994140625, -2.056640625, -0.10272216796875, 0.708984375, 0.0197296142578125, 0.77001953125, 0.82958984375, -0.6435546875, 0.9052734375, -0.95703125, -0.5439453125, -1.3623046875, -0.242919921875, -0.83837890625, 0.5908203125, 0.4365234375, -0.59765625, 0.935546875, -0.1749267578125, 0.17822265625, -1.65234375, 2.064453125, -0.892578125, 1.0146484375, -1.2470703125, 1.03515625, -0.83203125, 0.66845703125, -0.67041015625, -1.8310546875, -0.96435546875, 1.3671875, 0.7607421875, -0.407470703125, 1.6103515625, -0.2374267578125, 0.62744140625, 0.78759765625, -0.8994140625, -2.01953125, -0.329345703125, 1.013671875, 1.23828125, -1.9775390625, 0.20947265625, -0.65576171875, 0.469482421875, 1.666015625, 0.728515625, 0.08935546875, -0.1011962890625, 1.1005859375, -1.181640625, -0.171630859375, 0.2200927734375, 0.00518798828125, 0.8046875, -0.17578125, 0.95947265625, -1.06640625, -1.1240234375, 0.447021484375, 0.11175537109375, -0.25732421875, 0.421142578125, 0.93408203125, -1.091796875, -0.66796875, -1.62890625, 0.66162109375, 2.0390625, -2.478515625, 0.951171875, 1.5966796875, 1.1259765625, 0.1439208984375, 3.130859375, -0.08270263671875, -0.54443359375, 1.16015625, -0.6953125, -1.43359375, 0.353271484375, -0.1610107421875, -0.703125, 0.407470703125, -0.658203125, -2.119140625, -1.1142578125, 0.2052001953125, 0.479248046875, 0.63427734375, 2.50390625, -1.072265625, -0.6044921875, -0.5888671875, -0.203125, -0.51611328125, 0.4423828125, -1.1435546875, -1.90625, -0.79248046875, 0.73779296875, -1.216796875, -0.59912109375, 0.6201171875, -0.80615234375, -0.96875, -1.6015625, -1.783203125, 2.046875, 1.0849609375, 1.068359375, -0.62841796875, -1.658203125, 0.26806640625, -0.556640625, -1.521484375, 2.501953125, 1.6669921875, -0.83251953125, 0.24755859375, 1.775390625, -1.3271484375, -1.2919921875, -0.11474609375, 0.81298828125, -0.174560546875, 0.4130859375, 1.1201171875, -1.150390625, -0.01319122314453125, 0.1593017578125, 0.7509765625, 0.61669921875, 1.443359375, 0.82861328125, -1.220703125, -0.04205322265625, -0.861328125, 1.345703125, -1.3505859375, 1.2412109375, -0.5986328125, 0.5419921875, 0.56640625, -1.3369140625, 0.11572265625, -0.04547119140625, 0.26025390625, -0.53125, 0.1077880859375, 0.8623046875, -1.4697265625, -0.487548828125, 0.71630859375, 0.96826171875, 1.615234375, -1.2353515625, -0.3515625, -2.359375, -1.1435546875, 0.51025390625, 0.387451171875, -0.148681640625, 0.443115234375, -1.171875, -0.385498046875, 0.6142578125, 1.84375, 0.92138671875, 0.7060546875, -0.1556396484375, -1.49609375, -0.472412109375, -1.6591796875, 1.697265625, 0.425048828125, -1.5244140625, 1.7919921875, -1.4345703125, -0.435791015625, -1.189453125, 2.208984375, 0.353271484375, 0.2210693359375, 1.240234375, -0.13330078125, 1.6943359375, -0.55126953125, 0.333251953125, 0.60595703125, -0.28857421875, -1.150390625, 0.377197265625, 0.480712890625, -0.78515625, 0.51611328125, -0.1343994140625, -0.346923828125, 0.4931640625, -1.876953125, 0.12060546875, 0.61181640625, -1.09375, -0.845703125, -0.1295166015625, -1.353515625, -0.9619140625, 0.6533203125, -2.509765625, 0.40869140625, 0.5869140625, -1.091796875, 1.7421875, -1.2421875, -0.1602783203125, -1.7177734375, 1.8203125, 0.12890625, 0.12158203125, -1.169921875, -0.1068115234375, -0.53955078125, -0.0164794921875, 0.3994140625, -1.1455078125, 0.195068359375, -0.5888671875, -0.52001953125, -0.283203125, 0.73828125, 1.2548828125, -1.0087890625, -0.350830078125, 0.0157928466796875, 1.0810546875, -0.327392578125, -0.85498046875, -0.8466796875, 1.10546875, -0.1038818359375, 0.79736328125, -0.64990234375, -0.284423828125, -0.398681640625, 0.24462890625, -0.802734375, 1.6611328125, -0.17919921875, -0.5927734375, 1.705078125, -1.6611328125, -2.05859375, -0.241455078125, 0.34033203125, 0.1571044921875, -0.0191497802734375, -0.55419921875, -0.80224609375, -0.88037109375, -0.6650390625, 1.5498046875, 0.287109375, -0.362060546875, -0.0880126953125, 0.5634765625, 0.41845703125, -0.6875, -1.2900390625, 0.89111328125, 1.4638671875, -0.14501953125, 0.384521484375, 0.10491943359375, -0.80322265625, -1.3798828125, 1.373046875, 0.06488037109375, -0.247802734375, 0.36328125, 0.382568359375, 1.64453125, -0.2147216796875, 0.82958984375, -0.8408203125, 0.09625244140625, -0.5703125, -1.072265625, 0.9677734375, 0.10675048828125, 1.203125, -1.6708984375, 0.2119140625, -0.159912109375, 0.471435546875, -0.76171875, -0.2734375, 1.021484375, -0.32275390625, -0.8173828125, -0.52978515625, 0.70947265625, 0.6923828125, -0.038330078125, 0.0689697265625, 0.9306640625, -1.794921875, 0.54541015625, -0.3486328125, 0.6201171875, -2.47265625, 0.84521484375, 1.0966796875, -0.548828125, -1.2109375, -0.193115234375, 1.1572265625, 1.310546875, 2.748046875, 0.068115234375, 0.1236572265625, 0.8466796875, -1.6455078125, 0.052276611328125, 1.0634765625, -1.6015625, 0.39404296875, 0.9951171875, -1.9921875, -0.36669921875, -1.0546875, -0.330078125, -1.26171875, 0.205322265625, 0.4853515625, -1.5458984375, -0.72900390625, 0.525390625, 1.06640625, -2.70703125, 0.00933837890625, 1.091796875, 0.3876953125, 0.775390625, -0.497314453125, -0.90576171875, 0.39599609375, 0.62646484375, 2.080078125, 0.10980224609375, 1.447265625, 0.372802734375, -1.0947265625, -1.4140625, -1.2509765625, 0.020782470703125, 1.720703125, 0.1268310546875, 2.11328125, -1.3623046875, 0.62939453125, -0.07470703125, 1.126953125, -1.2548828125, -1.7138671875, 1.19921875, 0.80078125, -0.54345703125, -0.219970703125, -0.006084442138671875, 0.83642578125, -0.453125, -1.0595703125, -0.3310546875, -0.66259765625, 0.020050048828125, 1.3515625, -1.1298828125, 0.05841064453125, -0.81689453125, -2.595703125, 0.36376953125, -0.495849609375, 1.3291015625, -0.325439453125, 0.0286865234375, -1.8076171875, -0.0277557373046875, 0.03369140625, -0.434814453125, 0.59912109375, -0.162109375, 0.5234375, -0.0509033203125, -0.33203125, -1.091796875, -0.7421875, 0.2724609375, 0.47998046875, -1.11328125, 0.63427734375, -0.65625, 0.78955078125, 0.89306640625, 0.64013671875, 0.01107025146484375, 0.0004153251647949219, 0.59423828125, -0.49267578125, 0.00830078125, 0.26513671875, 0.6796875, -1.2646484375, 0.2451171875, -0.8798828125, 0.36767578125, -1.4462890625, 0.955078125, -0.1707763671875, -1.2802734375, 0.92431640625, -1.7451171875, -0.443359375, -0.87939453125, -0.94482421875, 1.482421875, 1.0498046875, 0.48046875, -0.274658203125, 0.192138671875, 0.859375, 0.050811767578125, 1.095703125, -0.59228515625, 0.09808349609375, -0.60205078125, 0.0174407958984375, -0.771484375, -1.029296875, 0.1312255859375, -0.72998046875, 1.2138671875, 0.10693359375, -0.92138671875, 0.030242919921875, -0.2379150390625, -0.98583984375, -0.34716796875, -0.1771240234375, 1.70703125, 2.087890625, -0.04827880859375, -0.1583251953125, 0.60595703125, 1.3828125, -0.19384765625, 0.1083984375, -0.17041015625, -0.91259765625, 1.224609375, -0.195068359375, -0.8798828125, 1.259765625, 0.30615234375, -0.492919921875, 0.40380859375, -0.452880859375, 0.21630859375, -0.382080078125, 0.384033203125, 0.962890625, 0.16845703125, 1.017578125, -0.697265625, 0.5537109375, 0.5419921875, -1.267578125, -2.544921875, 1.3349609375, 0.50048828125, 1.80078125, -1.130859375, 0.26708984375, 0.62939453125, -1.2080078125, -1.40625, -0.0540771484375, -0.71337890625, 1.8994140625, 0.61328125, 0.6611328125, -0.92822265625, 0.195068359375, -0.2467041015625, 0.304931640625, 0.08038330078125, -0.1385498046875, 0.9072265625, 0.69677734375, -0.78857421875, -0.8115234375, -0.268310546875, 0.69482421875, -1.9169921875, -0.170166015625, -0.58984375, 1.337890625, 0.52978515625, 0.40869140625, -1.48828125, 0.58984375, 1.3662109375, 0.64111328125, -1.720703125, 0.66259765625, -1.2626953125, -1.107421875, 0.0706787109375, 2.150390625, -0.619140625, 0.59814453125, 2.478515625, -0.2025146484375, -1.23828125, 0.188232421875, 0.2230224609375, 0.317138671875, 0.3408203125, -0.139892578125, 0.230712890625, 0.72265625, 0.7861328125, 1.1435546875, -0.04400634765625, 0.7861328125, 0.3740234375, 0.55859375, 1.1025390625, -0.396240234375, -0.76708984375, 0.7822265625, -0.59423828125, -0.544921875, -0.363525390625, -1.6962890625, 0.01335906982421875, 0.68017578125, -0.2388916015625, 0.00696563720703125, -1.2353515625, 0.3359375, -1.5009765625, 0.67138671875, -0.77587890625, 0.421142578125, 0.796875, -0.10400390625, 0.12176513671875, 0.72314453125, 0.966796875, -0.07269287109375, 0.080078125, 1.326171875, 0.92431640625, 1.12109375, 0.8623046875, -0.744140625, -0.41357421875, -0.463623046875, -0.13525390625, -0.68505859375, -0.2239990234375, -0.46435546875, 0.1728515625, -1.4970703125, -1.0546875, 0.7109375, 1.869140625, 0.30029296875, 0.9931640625, 0.78173828125, 0.09625244140625, 0.51611328125, -0.137451171875, 0.88330078125, 0.1251220703125, 0.88623046875, -0.1103515625, 1.8720703125, -0.58544921875, -0.174072265625, -0.65283203125, -1.0087890625, -0.236328125, -1.1181640625, 0.96533203125, 1.697265625, 1.5546875, -0.7470703125, -0.5126953125, -0.64208984375, 0.103271484375, 2.033203125, -0.98828125, -0.62548828125, -1.009765625, 0.5947265625, 1.2197265625, -0.018341064453125, 1.8603515625, -0.278076171875, -0.408935546875, 0.98095703125, -0.060699462890625, 1.0849609375, 1.501953125, 0.3818359375, -0.269775390625, -0.56640625, 1.91015625, 0.98779296875, 0.2183837890625, -0.685546875, -0.38818359375, -0.6669921875, -2.84375, 0.8837890625, 0.8623046875, 0.440185546875, 0.787109375, 0.292236328125, -0.77685546875, 1.2099609375, 0.403564453125, -0.54931640625, 2.04296875, -0.23779296875, -0.464111328125, -0.1302490234375, 0.27099609375, -0.218994140625, 0.124755859375, 1.0419921875, -0.0582275390625, -3.6640625, 0.014495849609375, 1.44921875, 0.36083984375, 0.06719970703125, 0.21923828125, -0.5634765625, -0.8701171875, 1.3623046875, 0.56494140625, 0.08636474609375, 1.0244140625, -1.0439453125, 0.440185546875, 1.0078125, -0.246337890625, 0.46044921875, 1.3037109375, 1.55859375, 1.181640625, 0.12548828125, -1.75, -1.177734375, 0.04754638671875, 0.1209716796875, 0.1517333984375, -0.294189453125, -1.5224609375, 1.337890625, 0.282958984375, 0.437744140625, -0.703125, -1.7880859375, 1.6650390625, 0.7255859375, -0.33056640625, 1.142578125, -0.9990234375, 0.2408447265625, -1.333984375, 0.10711669921875, -0.2008056640625, 0.9111328125, -0.7705078125, -1.0107421875, -0.2398681640625, -0.77392578125, 1.1875, 0.65673828125, -1.6650390625, 1.06640625, 0.623046875, -1.3212890625, 1.1103515625, 1.34765625, 1.26953125, -0.6162109375, -1.2978515625, 0.2347412109375, -0.39208984375, -1.5439453125, 1.40625, 0.65234375, -0.1590576171875, -0.74169921875, -0.107421875, -0.220458984375, 0.759765625, -0.446533203125, -0.8916015625, -1.478515625, -0.29443359375, -0.50830078125, -1.6943359375, 1.3115234375, 1.2294921875, 2.00390625, 0.7041015625, -1.1044921875, -0.1207275390625, 0.708984375, 0.28662109375, 2.140625, -0.74755859375, 0.759765625, -0.7939453125, 1.1953125, 0.68212890625, -0.1407470703125, 1.0625, -0.76318359375, -0.438720703125, -0.25244140625, -0.34619140625, 0.85888671875, -0.037445068359375, -1.0927734375, -2.513671875, 0.446044921875, 0.28369140625, -0.090087890625, 0.80078125, 0.5908203125, 0.587890625, 1.4013671875, -0.165283203125, -0.87646484375, 0.437255859375, -1.080078125, -0.30078125, 0.0838623046875, 0.41943359375, 0.58642578125, 0.57568359375, 2.404296875, -0.0882568359375, 1.6201171875, 1.8427734375, -0.5234375, -0.48681640625, -0.578125, -2.765625, -0.5205078125, 1.5322265625, -0.814453125, -0.433349609375, -0.63037109375, -0.383544921875, -0.08868408203125, -0.10260009765625, -0.4326171875, -0.08349609375, -2.16015625, -0.17529296875, 1.0029296875, -1.3828125, -0.75927734375, 1.7509765625, 0.0953369140625, 0.2266845703125, -1.19921875, -0.0249176025390625, -0.6708984375, 0.84619140625, -1.1767578125, 1.0576171875, 2.30859375, 1.052734375, -0.43701171875, -1.154296875, -0.744140625, -0.9091796875, 1.9326171875, -0.1328125, -1.3984375, -0.2122802734375, 0.6982421875, -0.472412109375, -0.162109375, -0.00806427001953125, 0.6533203125, -1.0009765625, 2.224609375, 1.1064453125, 0.708984375, 2.064453125, 1.0615234375, -1.1552734375, -1.724609375, 0.306640625, -3.271484375, 0.94970703125, -0.904296875, -0.98291015625, -0.2073974609375, 0.9990234375, -1.09375, -1.0283203125, -1.1650390625, -0.323486328125, -0.350830078125, -1.9384765625, 1.6474609375, -1.31640625, -0.5234375, 0.11309814453125, -0.91748046875, -0.26708984375, -0.94482421875, -1.013671875, -1.2265625, -0.494873046875, -0.9384765625, -0.66748046875, -1.3388671875, -0.71435546875, 0.75048828125, 0.76708984375, 0.70947265625, 0.02020263671875, 0.71630859375, 1.25, -1.0224609375, 0.10504150390625, -0.1475830078125, -0.990234375, 0.21240234375, 0.40625, -0.8544921875, 0.002086639404296875, 0.5380859375, -0.3271484375, 0.38525390625, 1.279296875, 0.0892333984375, 0.5849609375, 0.03607177734375, -0.27392578125, -1.453125, 1.1220703125, 0.76708984375, 0.39404296875, 0.296142578125, -1.0673828125, 0.9775390625, -0.06512451171875, 1.18359375, 0.99951171875, 0.283447265625, -1.0068359375, -1.0791015625, -0.00907135009765625, -1.0283203125, -1.568359375, -0.3388671875, 0.3125, 0.42822265625, 0.92529296875, -1.244140625, -0.291259765625, 0.658203125, -0.045379638671875, 0.23779296875, 1.099609375, 0.82568359375, 1.1943359375, -1.0869140625, -2.6484375, 1.0849609375, -0.025390625, 0.1142578125, 2.15234375, -1.0546875, 1.349609375, 0.391357421875, 0.6357421875, 0.4912109375, 0.10723876953125, -0.1319580078125, 0.78955078125, 2.224609375, 1.671875, -0.291748046875, -1.1181640625, -0.810546875, 0.318115234375, -1.23046875, -0.3720703125, 0.5400390625, 0.3701171875, 0.1260986328125, -0.89013671875, -0.1678466796875, 1.1650390625, 0.400634765625, 1.689453125, 1.4453125, 0.80078125, 0.7734375, -0.55419921875, -0.1986083984375, 0.029998779296875, 1.3515625, 0.89794921875, 1.0126953125, 0.4013671875, -0.00959014892578125, 0.55517578125, -0.39599609375, -0.68603515625, 0.65966796875, -2.640625, -0.55615234375, 0.28564453125, -1.28515625, -2.3046875, 1.24609375, 0.5908203125, 2.119140625, 0.82861328125, -1.3212890625, 1.162109375, 0.16064453125, 0.2254638671875, 1.1435546875, 1.0068359375, -0.81591796875, 1.240234375, -1.646484375, 1.033203125, -0.1795654296875, 0.393310546875, -0.69482421875, -1.865234375, -0.3427734375, -0.459228515625, -0.78466796875, 0.419921875, -0.2734375, -1.1982421875, 0.340087890625, -0.56982421875, -0.89892578125, 1.296875, 0.1815185546875, 0.7197265625, 0.9150390625, -1.5283203125, 0.62939453125, -1.3779296875, -0.44189453125, 0.96142578125, 0.233642578125, 0.1268310546875, 1.0595703125, 0.68701171875, 0.73291015625, 2.19140625, -0.31298828125, -0.5576171875, 0.94287109375, 0.90234375, -0.67236328125, -1.03515625, -0.1334228515625, -0.1395263671875, -0.300048828125, 1.306640625, 1.05078125, 0.5849609375, -0.375244140625, -0.46337890625, 0.8115234375, -0.253173828125, 1.2412109375, 0.2152099609375, -0.44970703125, 2.01953125, -0.4765625, 0.2232666015625, -0.46826171875, 0.2386474609375, -0.2890625, 1.2529296875, -0.79931640625, 0.339599609375, -0.1531982421875, 0.134521484375, 0.261962890625, -1.8623046875, -1.17578125, 0.94580078125, 0.7353515625, 1.2783203125, -1.3193359375, -0.1220703125, -0.69091796875, -0.81884765625, -0.280517578125, -0.41748046875, 0.1904296875, 0.69677734375, -0.00630950927734375, 1.4638671875, -1.4697265625, 0.04254150390625, 1.90234375, -0.57568359375, 0.2418212890625, -0.47900390625, -0.361328125, -0.314697265625, 1.390625, 0.85400390625, 0.0736083984375, 0.46435546875, -1.2109375, -1.59375, 0.564453125, -0.7666015625, -0.463134765625, 0.0258941650390625, 0.21044921875, -1.84375, 1.021484375, 1.068359375, -0.52880859375, 1.0244140625, 2.19140625, -0.09820556640625, 0.35498046875, -0.435546875, 1.2802734375, -0.2490234375, -1.6123046875, 0.41748046875, -0.04644775390625, 0.0765380859375, 0.5830078125, 0.51025390625, 0.34814453125, -0.1982421875, -0.29736328125, 0.6083984375, 1.041015625, 1.3427734375, 0.67138671875, 0.8046875, -0.68115234375, -2.267578125, -0.7265625, -1.0380859375, -0.1793212890625, -0.323486328125, 0.5849609375, 0.52001953125, 1.0322265625, -0.72314453125, 0.9521484375, 0.6923828125, 0.315185546875, -0.339599609375, -0.2244873046875, 0.60546875, -0.5234375, 0.334716796875, -0.110107421875, -0.26220703125, -1.34375, 0.7939453125, 1.072265625, 0.56396484375, 0.216552734375, -0.8125, -1.728515625, 0.252685546875, -0.77685546875, 0.481201171875, 0.6904296875, -1.64453125, -0.61669921875, -0.75146484375, -0.34716796875, 0.11785888671875, 2.359375, -0.76806640625, 0.6337890625, 1.056640625, 0.6796875, 1.3798828125, -2.181640625, 0.5927734375, 0.40966796875, -0.0214691162109375, -0.5615234375, -0.30029296875, 1.232421875, -1.76953125, -0.80419921875, -1.3671875, 2.345703125, -0.99609375, -0.80712890625, -0.1005859375, 0.254150390625, -1.080078125, -0.3505859375, 0.352783203125, 0.46533203125, -0.89697265625, -1.234375, -0.5927734375, 1.6083984375, -0.77392578125, 0.239013671875, -0.8955078125, 1.1005859375, 1.255859375, 0.86474609375, 0.459716796875, -0.849609375, -1.0947265625, -0.68505859375, -0.43115234375, -0.7412109375, -0.401123046875, 0.5048828125, -1.6533203125, -1.72265625, 1.4052734375, 0.78662109375, -0.5517578125, -0.208740234375, -0.47705078125, -0.318603515625, 0.26220703125, -1.197265625, -0.0869140625, -1.1552734375, 0.71875, -0.2578125, -0.059234619140625, -1.0, -1.5703125, 1.7822265625, 0.6513671875, 0.7080078125, 1.130859375, -1.330078125, 0.00611114501953125, -1.34375, 0.59423828125, -0.7109375, -0.45654296875, 0.170654296875, 0.62451171875, -0.89501953125, 0.358154296875, -0.298583984375, 0.61865234375, 0.0909423828125, 0.82373046875, -0.66455078125, -0.07489013671875, 0.1973876953125, 0.5390625, -0.1954345703125, -0.94287109375, -0.060638427734375, 1.73046875, -1.04296875, 0.5654296875, 1.3544921875, 0.41162109375, 1.064453125, -0.5791015625, 0.7900390625, 1.0986328125, 2.408203125, -0.05499267578125, -0.04473876953125, -0.7587890625, -0.2274169921875, -0.1834716796875, -0.59716796875, -0.6142578125, -0.39794921875, 0.320556640625, 0.48828125, -0.292236328125, -0.69091796875, 0.7421875, -0.2493896484375, -1.7021484375, -0.31005859375, 0.54833984375, -1.328125, -1.6162109375, 0.91162109375, -0.461181640625, -0.32373046875, 0.4677734375, -0.78564453125, 0.44140625, -0.40185546875, 0.8505859375, 0.5498046875, -0.74267578125, -0.229248046875, 0.83642578125, -0.306884765625, 0.7939453125, 0.430908203125, -0.85107421875, 1.08203125, -0.023529052734375, 1.1640625, 1.0390625, -1.001953125, 0.389404296875, -0.1260986328125, -0.09368896484375, 1.3701171875, 2.599609375, 0.15234375, 0.1534423828125, 1.4658203125, 0.51123046875, -1.0126953125, -0.1337890625, -0.60986328125, 0.521484375, -1.552734375, -1.65625, -0.01355743408203125, 1.0400390625, -0.72216796875, 1.3017578125, -1.2158203125, 0.90283203125, -1.1875, 0.00567626953125, -0.72998046875, -0.2025146484375, 1.2041015625, -0.50146484375, -0.66943359375, -2.484375, 0.73876953125, 2.349609375, 0.4775390625, 0.26806640625, -0.07672119140625, 0.86962890625, 0.5322265625, 1.2041015625, -1.0634765625}; - -static const __fp16 Bpack_dram[8192] - __attribute__((section(".dram"), aligned(64))) = {0.2015380859375, -0.08856201171875, -0.9208984375, -1.4931640625, 1.306640625, -0.580078125, 0.72265625, -1.2666015625, 1.650390625, 0.371337890625, -1.4228515625, -0.335693359375, 0.55908203125, 0.07427978515625, 0.2166748046875, -0.6416015625, 1.1162109375, 0.32177734375, -0.6826171875, -0.1304931640625, -0.23583984375, -0.5634765625, -1.4921875, 0.87841796875, 1.4365234375, -1.6533203125, 0.158935546875, 0.74365234375, -2.177734375, -0.6318359375, -0.289794921875, -0.96875, 0.9951171875, 1.2587890625, 0.0860595703125, 1.576171875, 0.0650634765625, -1.03515625, 0.5439453125, -1.0263671875, 2.05078125, 2.130859375, -0.55517578125, -1.158203125, -1.162109375, -1.6201171875, 2.005859375, 0.4990234375, 1.287109375, -1.2783203125, 0.2401123046875, 0.1312255859375, 0.285888671875, 0.072998046875, -0.326904296875, -0.11248779296875, 1.3544921875, 0.300537109375, -0.916015625, -0.70703125, 0.52001953125, 1.1513671875, -0.607421875, -0.77685546875, -0.919921875, 0.25927734375, 0.91455078125, -0.11859130859375, 0.033355712890625, 0.03741455078125, -0.77880859375, -0.349853515625, -2.0390625, 0.57470703125, 0.77734375, 1.3447265625, -0.1419677734375, 0.88037109375, -1.5732421875, -0.57373046875, 1.4072265625, 0.75048828125, -1.083984375, 1.138671875, -0.36279296875, -0.833984375, 1.0263671875, 0.98486328125, 1.6943359375, 1.2021484375, 1.005859375, -0.56982421875, 0.2509765625, 0.41748046875, -1.533203125, -0.73779296875, 1.3916015625, 0.7216796875, 2.029296875, -0.48193359375, 0.884765625, -0.088134765625, 0.609375, -0.92333984375, 0.77685546875, 1.7724609375, 0.2259521484375, 0.6337890625, 0.6083984375, 0.58740234375, 0.3916015625, -0.97900390625, -0.66650390625, 0.74658203125, 0.7900390625, -1.6455078125, -0.330322265625, 0.6318359375, -0.05181884765625, -0.148681640625, -0.36376953125, -0.53125, -0.05828857421875, 2.16015625, 0.5244140625, 1.76171875, 1.6435546875, 0.69921875, -0.1258544921875, -1.4638671875, 1.357421875, -0.880859375, -0.079833984375, -0.8876953125, -1.5224609375, 1.677734375, 0.6435546875, -0.7451171875, -0.416015625, -0.260009765625, 0.297607421875, 0.5458984375, -1.6083984375, 0.587890625, 1.3125, 0.65966796875, -1.5771484375, -0.3564453125, -0.16748046875, 0.006435394287109375, -0.334228515625, 0.024169921875, 0.496337890625, -1.2890625, -0.023193359375, 0.7119140625, -0.299560546875, -1.33203125, 0.1378173828125, -1.8125, 0.5478515625, 1.9150390625, 0.05035400390625, -0.671875, -0.084716796875, 1.2490234375, -1.4912109375, -0.94091796875, -0.9638671875, -0.40771484375, 1.7802734375, 1.36328125, 0.6982421875, -0.8037109375, -1.65625, -0.720703125, -1.3466796875, 1.7021484375, -0.87841796875, -1.1767578125, 0.8232421875, -0.07891845703125, 0.34423828125, -0.80224609375, 1.7353515625, 2.23828125, -0.01320648193359375, -0.5634765625, -0.99267578125, 1.4580078125, 0.073974609375, -0.366455078125, 0.1591796875, 1.876953125, -0.6181640625, -0.298095703125, 1.515625, -0.26318359375, -1.216796875, -1.359375, 0.0236968994140625, 0.37255859375, -0.1871337890625, -0.2734375, 0.28271484375, 1.1845703125, -1.646484375, -1.076171875, 1.0302734375, -2.716796875, -1.10546875, 0.90771484375, -1.3525390625, 1.4423828125, -1.0078125, -0.308349609375, -0.35986328125, -0.6044921875, 0.273681640625, 1.1787109375, -1.4189453125, -0.450439453125, 0.2705078125, -0.1461181640625, -0.603515625, -0.2122802734375, 1.0234375, 1.0166015625, 0.982421875, 1.4453125, -0.10577392578125, 0.062744140625, 1.6630859375, -1.033203125, -1.244140625, 0.2139892578125, -0.40673828125, -1.2236328125, 0.88818359375, 1.482421875, 0.196533203125, 0.78125, 0.51611328125, 1.767578125, 0.338623046875, 1.8720703125, -0.006053924560546875, 0.806640625, -1.8603515625, -0.43994140625, -0.2484130859375, 0.599609375, 0.72314453125, -1.94140625, -0.76025390625, 0.391357421875, 0.93994140625, -0.06256103515625, -0.5869140625, 1.0205078125, 0.50634765625, -1.13671875, 1.1650390625, 1.7861328125, -0.06060791015625, -1.2626953125, 0.8515625, -0.70654296875, 0.7021484375, 0.3291015625, -0.202880859375, 0.140869140625, 0.65234375, -0.7109375, -0.273681640625, -2.1640625, 2.111328125, -0.27880859375, 1.6494140625, -0.89501953125, 0.366455078125, 1.0498046875, -0.0865478515625, 1.765625, 1.69921875, -0.460205078125, -0.3515625, -1.2138671875, 0.62548828125, -0.378662109375, 0.185302734375, -0.128173828125, -0.53564453125, 0.27294921875, -1.1767578125, 0.91796875, 0.023345947265625, 1.072265625, 0.79345703125, 1.9130859375, -2.11328125, 0.3642578125, -0.1734619140625, 0.41259765625, -0.280029296875, 1.0751953125, 1.19140625, 0.341796875, -1.099609375, 1.4267578125, 0.16796875, -0.473388671875, -0.9189453125, -1.5673828125, -0.372802734375, 1.72265625, -0.732421875, 1.21875, 1.884765625, -1.6171875, 0.1314697265625, 0.1954345703125, -1.42578125, 0.60009765625, 0.54443359375, -0.78369140625, -0.292236328125, -0.892578125, -0.032745361328125, -0.26904296875, 0.69482421875, -1.9599609375, -0.87890625, -0.10809326171875, 0.5732421875, -0.3115234375, -0.1978759765625, 0.30908203125, -0.63232421875, 1.119140625, 0.81640625, -1.7021484375, 0.1192626953125, -1.3828125, -0.269775390625, 0.57568359375, 0.9482421875, -0.828125, 0.1949462890625, 0.90576171875, 0.65380859375, -0.201171875, 0.93505859375, -0.3037109375, 1.91796875, -0.6923828125, -0.214599609375, -1.5625, 0.81005859375, -0.73486328125, -0.1201171875, 0.47607421875, -1.5478515625, 0.380615234375, 1.474609375, 1.1474609375, -0.2646484375, -1.400390625, -0.6181640625, 0.572265625, -1.6953125, 0.427734375, -0.365234375, 1.19140625, -1.111328125, 0.283935546875, -0.0919189453125, 0.2939453125, 1.125, 0.90478515625, 0.333251953125, -0.164306640625, 0.482421875, -0.1544189453125, 0.8037109375, -1.2099609375, -0.96337890625, 0.22705078125, -0.1895751953125, 1.953125, 0.65625, 0.0709228515625, 0.78955078125, -0.0164794921875, -0.484619140625, -0.6201171875, 0.0765380859375, 0.484130859375, -0.78564453125, -0.4228515625, 0.591796875, 0.70751953125, -0.74609375, 1.0947265625, -1.0771484375, -1.6484375, 1.1474609375, 0.9765625, 0.333740234375, -0.9580078125, 0.73876953125, 1.86328125, 0.78662109375, -1.5302734375, 2.04296875, -0.5859375, -0.951171875, -2.041015625, 1.3876953125, -0.1685791015625, 0.541015625, 1.568359375, -1.0712890625, 0.42724609375, -0.09539794921875, -1.1552734375, 0.454833984375, -0.5, -1.9931640625, 0.81201171875, 0.80126953125, 0.07366943359375, -0.900390625, 2.20703125, -0.1700439453125, -0.431396484375, 0.2401123046875, 1.724609375, -1.22265625, -0.51025390625, -1.6650390625, 1.576171875, -0.85302734375, -0.4658203125, 0.93212890625, 0.7451171875, -0.04400634765625, -0.865234375, 1.2236328125, -2.087890625, 0.65478515625, 1.7216796875, -0.13818359375, 0.41162109375, 1.3564453125, -0.63330078125, 0.98583984375, -0.317626953125, -0.529296875, -0.07037353515625, 0.90380859375, -0.41650390625, -0.892578125, 0.8125, 0.9599609375, -0.61083984375, 0.44384765625, 0.1553955078125, -1.44140625, -0.390869140625, 0.1915283203125, 1.4423828125, 0.88232421875, 0.72314453125, 0.366943359375, 0.253662109375, -0.041473388671875, -0.626953125, -0.0067596435546875, 0.71240234375, -0.433349609375, 0.7890625, 0.0025787353515625, -0.6552734375, -0.98486328125, 0.63037109375, -0.06500244140625, -0.1307373046875, 1.1962890625, -0.39697265625, 0.0517578125, -0.8828125, -1.041015625, 1.0146484375, 1.556640625, -1.146484375, -0.79052734375, -0.312255859375, 0.7890625, 1.0234375, 0.0142059326171875, -0.2476806640625, 0.54541015625, 0.07708740234375, -0.1451416015625, 1.3994140625, -0.08746337890625, -0.6708984375, -1.1416015625, -1.130859375, -0.125732421875, -0.46337890625, 0.2237548828125, 0.98974609375, 0.1812744140625, 0.5830078125, 0.94287109375, 0.15234375, 0.46923828125, 0.34716796875, 0.0966796875, 0.48779296875, 0.230224609375, -0.0850830078125, 1.056640625, 0.384033203125, 1.962890625, 1.1728515625, 0.56982421875, -0.73876953125, -0.484375, -0.8115234375, -2.30859375, -1.58203125, -0.3544921875, 0.315673828125, -2.34765625, 0.2174072265625, 1.4091796875, 0.13916015625, 0.11273193359375, -1.302734375, -1.25390625, -0.9326171875, 0.084716796875, 0.583984375, 0.86279296875, -0.034576416015625, 0.07501220703125, -1.6171875, 0.08013916015625, -0.72607421875, 1.7265625, 0.59033203125, 0.669921875, 0.04583740234375, 0.77099609375, 1.3408203125, -1.095703125, -1.7998046875, 1.02734375, -0.54296875, 0.151611328125, -0.57568359375, 1.224609375, -0.264892578125, 0.603515625, -0.59033203125, 0.80322265625, 1.5546875, -0.55712890625, -1.689453125, 1.1083984375, 1.41015625, -1.0341796875, 1.345703125, 0.21044921875, -0.454345703125, -1.7685546875, -1.1123046875, 1.64453125, -0.8818359375, -2.3125, -0.347412109375, -0.447509765625, 0.458740234375, 0.05694580078125, -1.5087890625, 0.50341796875, -0.1053466796875, 0.34033203125, -0.3662109375, -0.99951171875, -1.068359375, -0.89111328125, 0.421875, 0.7939453125, 1.568359375, 0.70849609375, -0.4306640625, 0.64111328125, -0.0311279296875, -0.1680908203125, -1.1689453125, 0.84912109375, -0.2169189453125, 0.1298828125, -0.3369140625, 0.275146484375, 1.197265625, -0.75341796875, 0.1181640625, -1.6103515625, 1.1826171875, -0.98779296875, -1.23046875, 0.0296783447265625, -0.004314422607421875, 0.10687255859375, 0.6875, 1.0595703125, -1.4931640625, -1.837890625, -0.09246826171875, 0.89111328125, -1.1611328125, -0.5498046875, -0.0682373046875, -0.1700439453125, -0.00022423267364501953, 0.42724609375, 0.4931640625, -1.9453125, 0.54248046875, -0.02496337890625, 0.9150390625, -0.37158203125, 1.4326171875, -2.345703125, -1.720703125, -1.49609375, 1.390625, 0.0445556640625, -0.76025390625, 1.8115234375, 0.990234375, 0.464111328125, 0.08740234375, -0.30126953125, 2.306640625, -0.4013671875, 0.379638671875, -0.8291015625, -0.15087890625, -0.71923828125, 0.1649169921875, 0.262451171875, -0.05413818359375, 1.0859375, 1.6728515625, -0.365234375, 0.09600830078125, 1.5048828125, -1.197265625, -0.66259765625, 1.2783203125, 1.1279296875, 0.2017822265625, 0.56005859375, -0.75927734375, 0.7080078125, -2.287109375, 0.09600830078125, 0.1453857421875, 2.0859375, -1.0205078125, 0.61572265625, -0.459228515625, 1.27734375, -1.4375, 0.830078125, 1.03515625, -1.0107421875, 1.14453125, 0.5361328125, -0.1995849609375, 0.26416015625, -1.126953125, -1.12109375, 0.4892578125, 0.27783203125, 0.60546875, 1.421875, 0.11834716796875, -0.64892578125, 0.71337890625, 1.78125, -0.5185546875, -2.306640625, -0.48046875, 0.4248046875, -1.7314453125, 1.515625, 0.38134765625, 0.218994140625, 0.8232421875, 0.14111328125, -0.7734375, 0.93408203125, 2.703125, 0.144775390625, -0.406005859375, 0.08984375, -0.97705078125, 1.306640625, 0.357177734375, -1.84375, 1.3486328125, -0.78564453125, -0.09954833984375, 0.238525390625, 0.96484375, 1.00390625, -2.37890625, 1.0947265625, -0.2332763671875, -1.904296875, 0.66748046875, -0.52783203125, -0.57373046875, 0.6689453125, 0.76025390625, 1.5302734375, 1.541015625, 0.525390625, -0.454345703125, -1.2021484375, -1.0625, -0.978515625, -1.0224609375, 1.046875, -0.08111572265625, 0.7509765625, 1.2197265625, -0.251708984375, -0.37646484375, 0.892578125, -0.7119140625, -0.1759033203125, -1.6591796875, -1.6689453125, -0.63623046875, -0.57080078125, -0.052825927734375, 1.6455078125, 0.290283203125, 0.4013671875, 2.5625, 0.385986328125, -1.599609375, 0.86279296875, -0.0938720703125, -0.004367828369140625, -0.158203125, -1.0107421875, 0.3896484375, 1.0029296875, 0.05157470703125, 1.072265625, 0.470458984375, 0.382080078125, 0.441650390625, 0.0341796875, 0.60986328125, 0.265625, -0.70166015625, -0.75830078125, -0.85302734375, 0.90673828125, 0.85986328125, 0.1357421875, 0.69482421875, -1.9892578125, 1.8671875, -0.6669921875, 0.982421875, 0.283447265625, 0.1397705078125, 0.24072265625, 2.505859375, -2.251953125, 0.308837890625, -0.427490234375, 0.1287841796875, -0.75732421875, -1.7177734375, -1.1416015625, -1.2353515625, 0.99560546875, 0.431640625, 0.07696533203125, -0.34521484375, 0.34716796875, 0.74658203125, -1.41015625, 0.5712890625, 0.837890625, -0.12548828125, 1.2080078125, 1.0380859375, 0.0927734375, -0.76708984375, 0.1546630859375, -1.580078125, -1.0576171875, -0.99609375, -1.1220703125, 0.1318359375, 0.89208984375, 0.175537109375, 0.15625, -0.1688232421875, -0.2408447265625, -0.52001953125, 0.720703125, 1.6767578125, 0.81884765625, -0.11407470703125, 0.033782958984375, -1.1328125, 2.310546875, -0.286376953125, 1.89453125, -0.51318359375, -0.55029296875, 1.6337890625, 1.0009765625, 0.578125, 0.166259765625, 1.0615234375, 1.0048828125, -0.44677734375, -0.60009765625, -0.050140380859375, 1.1591796875, 2.103515625, 1.7177734375, -0.154541015625, -0.285400390625, 1.5517578125, 0.49853515625, -1.220703125, -0.708984375, -1.9560546875, 0.8251953125, 0.295166015625, -1.45703125, 0.92919921875, 1.1875, -1.3662109375, 0.57763671875, -0.267578125, -1.6083984375, -2.876953125, 0.98291015625, 0.3212890625, -1.8154296875, 0.71240234375, 0.55810546875, -0.841796875, 0.08172607421875, -0.455078125, -0.4873046875, -0.97705078125, 2.236328125, -2.21484375, -0.6826171875, -0.1575927734375, -0.51611328125, -1.2490234375, 0.015594482421875, -0.5029296875, -0.89306640625, 1.0654296875, 0.166748046875, -0.63525390625, -0.051483154296875, -0.240234375, -1.4072265625, -0.72998046875, -1.7412109375, 1.5615234375, -1.9736328125, 0.666015625, -0.74658203125, 0.2249755859375, 1.5810546875, 0.7119140625, -2.474609375, -0.334716796875, 1.4072265625, 0.360595703125, 0.1956787109375, 1.6279296875, -0.3876953125, 0.8779296875, -0.45068359375, 0.29296875, 1.3505859375, 0.91552734375, 0.019317626953125, -1.0517578125, -0.335205078125, -0.07305908203125, -0.6962890625, -0.66455078125, -0.192138671875, 0.8564453125, 1.3974609375, 0.1102294921875, -0.042999267578125, 0.70947265625, 0.014923095703125, 0.031707763671875, 0.430419921875, -0.145263671875, 1.3115234375, -0.1595458984375, 0.10125732421875, -1.1552734375, 1.115234375, 1.0009765625, -0.384765625, -1.0458984375, 1.513671875, 0.491943359375, 1.818359375, -0.67724609375, 0.37109375, -1.6875, -1.1845703125, 0.45068359375, -2.55859375, -0.004779815673828125, -0.18701171875, 1.158203125, 0.493896484375, -0.7529296875, -0.2154541015625, -0.405517578125, -1.82421875, -0.43115234375, 0.384765625, 1.9033203125, 0.302978515625, 0.329833984375, -0.58544921875, 0.297607421875, -0.279541015625, -0.51708984375, -0.347412109375, -1.18359375, -0.371337890625, 0.90966796875, -1.3310546875, 1.0478515625, -1.6513671875, -0.7001953125, 0.0093231201171875, -1.537109375, -1.0439453125, -0.837890625, 0.4951171875, -0.705078125, 0.363525390625, -0.54931640625, -0.2303466796875, -1.724609375, -0.55078125, 2.08984375, 1.0166015625, -0.10516357421875, -0.640625, -0.12078857421875, 0.7080078125, 0.409423828125, 1.013671875, 0.64990234375, -0.284423828125, -0.87353515625, 0.16015625, -2.119140625, 0.82177734375, -1.421875, 0.72021484375, -2.298828125, -0.65380859375, 0.63623046875, 2.056640625, -1.12109375, 0.60498046875, -0.576171875, -0.259521484375, -0.2203369140625, 0.153564453125, 1.322265625, 0.74755859375, 1.6806640625, -0.251220703125, 0.2210693359375, -0.05706787109375, -0.4970703125, 0.411865234375, 0.5048828125, 0.60791015625, 0.759765625, 1.81640625, -0.406494140625, -0.1070556640625, -0.43310546875, -0.96484375, -0.76025390625, -0.038055419921875, -0.328369140625, 2.220703125, -0.3271484375, -1.32421875, -3.103515625, -1.515625, -2.2421875, 0.44140625, -1.2412109375, 0.264404296875, -2.509765625, 0.2315673828125, -1.4677734375, -1.384765625, -1.3154296875, -0.38525390625, -0.347900390625, -0.193115234375, 0.388671875, 0.309814453125, -1.046875, -0.0933837890625, 2.1640625, 0.1243896484375, 0.55029296875, -1.6044921875, -0.5419921875, -0.94482421875, 2.16015625, 0.9765625, 0.63427734375, 0.53466796875, -1.1357421875, -1.107421875, 0.919921875, -0.84033203125, -0.234130859375, -1.615234375, -0.84375, 1.458984375, -0.802734375, -1.05078125, -0.88037109375, 0.373046875, 0.365478515625, -0.55810546875, -0.61865234375, -0.0262603759765625, -1.1337890625, 1.3369140625, -0.1451416015625, -0.568359375, -0.31640625, 1.1533203125, -0.324951171875, 0.2376708984375, -0.062103271484375, -0.39013671875, -1.09765625, -1.703125, -1.6123046875, -0.270751953125, 0.54052734375, -0.7587890625, -0.1832275390625, 0.54833984375, 0.1998291015625, -0.8046875, -0.80078125, 0.1475830078125, -0.6494140625, -1.6083984375, 0.4150390625, 1.0576171875, -0.73583984375, -0.2646484375, 0.95556640625, 0.80908203125, 2.037109375, 1.0732421875, -0.7294921875, 1.6669921875, -0.6455078125, 0.5458984375, 0.27490234375, -0.970703125, -0.330322265625, 0.72314453125, -1.259765625, -0.423828125, -0.10186767578125, -0.76416015625, 0.4794921875, 1.462890625, -0.428466796875, -0.31591796875, -1.5673828125, 0.425048828125, -1.6591796875, 0.489990234375, 0.192138671875, 0.59716796875, 2.03125, 0.150146484375, 0.01168060302734375, -0.304443359375, -1.126953125, -0.5810546875, -0.4208984375, -0.299560546875, 0.71484375, -0.62109375, -0.94873046875, 0.1461181640625, -0.5048828125, 0.2666015625, -0.450439453125, -0.332275390625, 0.06134033203125, 0.004329681396484375, 0.38427734375, -0.18212890625, -0.1435546875, -0.5244140625, -0.326171875, 0.77294921875, 2.474609375, -0.483642578125, 1.251953125, 1.720703125, -0.360595703125, -1.6689453125, 1.255859375, 1.5126953125, -0.32958984375, -0.0526123046875, -0.395263671875, 0.6708984375, -0.412353515625, -0.146484375, -0.12225341796875, -0.98095703125, 1.3984375, 0.853515625, -0.9794921875, 0.96044921875, -0.9951171875, -0.5712890625, 2.03515625, 1.4580078125, 0.8212890625, -0.841796875, -0.81640625, 1.4453125, -1.599609375, 0.41796875, 1.015625, 0.441650390625, -1.69140625, -0.5517578125, -0.488037109375, -0.022003173828125, 1.1083984375, 0.79443359375, -0.032470703125, 0.1910400390625, -1.6650390625, -2.763671875, 0.7099609375, 0.66455078125, 0.779296875, 0.416015625, 0.60791015625, 0.8818359375, -0.43798828125, -0.9716796875, 0.951171875, 0.5205078125, -0.128173828125, 1.3623046875, -0.90087890625, 1.029296875, 1.7998046875, 2.34765625, -0.26708984375, -0.71630859375, -0.6953125, 0.63818359375, -0.45263671875, -1.0703125, 0.446533203125, 0.1119384765625, -2.04296875, -1.076171875, 0.42919921875, 0.76904296875, 1.037109375, -0.271240234375, 1.0, 0.1351318359375, 0.29052734375, -1.255859375, 1.779296875, -0.334228515625, 0.521484375, 0.09820556640625, 0.587890625, 1.2421875, -0.240234375, -0.0836181640625, 0.4267578125, -0.74267578125, -0.552734375, 0.1322021484375, -0.1427001953125, 0.51953125, -0.3896484375, -0.059326171875, -0.482421875, -0.5244140625, 0.038665771484375, -0.2064208984375, 0.21533203125, -0.0699462890625, -1.482421875, -0.78369140625, 1.34765625, 0.80078125, -0.89208984375, 0.36865234375, 0.10150146484375, -0.60009765625, 1.3427734375, 0.1356201171875, -1.4443359375, 0.351806640625, -0.80126953125, 1.62890625, -0.443603515625, 2.1640625, 0.041168212890625, -0.0693359375, -1.4619140625, 0.168212890625, 1.3720703125, -0.50146484375, 1.5126953125, -1.515625, -0.38623046875, -0.34130859375, -1.0009765625, 0.496826171875, 0.962890625, 3.080078125, 1.279296875, 0.041107177734375, -1.4208984375, -0.2076416015625, -0.11126708984375, 0.0272674560546875, -1.240234375, -0.65625, -1.3984375, -1.6728515625, -1.0537109375, -0.381591796875, -0.5830078125, 0.73291015625, 0.041839599609375, 1.775390625, -0.5576171875, -0.32080078125, 0.69580078125, 2.025390625, 0.216796875, -1.1845703125, -1.744140625, -0.70703125, -0.71826171875, -0.76220703125, 0.0838623046875, -0.061737060546875, 0.51708984375, 0.751953125, -0.316162109375, -0.3125, -0.304443359375, 1.2421875, -0.259033203125, -0.176513671875, -1.8017578125, -1.4482421875, 0.9404296875, -1.275390625, 1.3212890625, -0.95849609375, -0.45361328125, 1.5322265625, 0.70849609375, -1.75, 0.54296875, -0.798828125, 0.86328125, 0.429443359375, -0.86767578125, -1.1865234375, -1.6708984375, 0.60986328125, -0.3447265625, 0.067138671875, -1.537109375, -0.1142578125, -0.222412109375, 0.61328125, -1.0078125, -0.99853515625, -1.2578125, -0.1854248046875, 0.5439453125, 1.130859375, -0.66796875, -1.041015625, -0.84130859375, -0.64599609375, 1.37890625, -1.5908203125, -0.666015625, 1.5009765625, -0.9140625, 0.46044921875, -0.16455078125, -0.50146484375, 0.69287109375, -0.67236328125, -1.1201171875, 0.06134033203125, -1.2255859375, -0.7685546875, 0.046173095703125, 1.49609375, -1.1416015625, 0.336181640625, 1.029296875, 0.72412109375, -0.8603515625, 1.4638671875, -1.234375, -0.263916015625, -0.56982421875, 0.454833984375, 0.0687255859375, 0.0841064453125, -0.00649261474609375, 0.94287109375, -1.1923828125, -0.625, -0.42138671875, 0.8251953125, 3.044921875, -0.580078125, 0.716796875, -0.7900390625, -0.0731201171875, 1.5361328125, -0.024017333984375, -0.0243682861328125, 0.93408203125, -1.521484375, 3.84765625, 0.6259765625, -0.19580078125, -1.0458984375, -0.974609375, -0.1981201171875, 0.2314453125, -1.4189453125, 1.5986328125, 0.326904296875, -0.1214599609375, -0.489501953125, 0.57421875, -1.103515625, -1.12109375, 0.08135986328125, 0.1043701171875, 1.7724609375, 0.7490234375, 1.0078125, -0.298095703125, -1.748046875, 0.83154296875, -2.7421875, -1.3583984375, 0.4755859375, 0.60693359375, -0.60302734375, 0.71533203125, -0.466796875, 0.40625, -1.2802734375, 0.7783203125, -0.181396484375, 1.1513671875, -0.94140625, 1.244140625, -0.6708984375, -1.728515625, -0.8642578125, -0.556640625, -0.70703125, -1.330078125, -2.13671875, -0.7236328125, -0.76611328125, 1.716796875, 1.421875, 0.4130859375, 0.2244873046875, 0.58251953125, 0.6630859375, -1.0859375, 0.60693359375, -0.9619140625, -0.4443359375, -1.498046875, -0.12939453125, -0.640625, 0.133056640625, -0.61328125, -0.054656982421875, 0.338134765625, 1.0302734375, 1.1220703125, -0.7021484375, 0.333984375, -1.310546875, -0.486083984375, -1.3017578125, 0.611328125, 0.52392578125, 1.732421875, -0.7021484375, 0.96533203125, -0.6669921875, -0.09381103515625, -0.057037353515625, 0.456298828125, -2.064453125, -0.59521484375, -1.603515625, 0.90087890625, -1.4638671875, 0.650390625, 0.57763671875, 1.7939453125, -1.3046875, 0.1605224609375, 1.8974609375, -0.23193359375, -1.6015625, -2.0703125, 0.5859375, 0.329345703125, 0.3818359375, 0.199462890625, 0.038787841796875, 0.66162109375, -1.6904296875, -1.1884765625, -0.37158203125, 0.92333984375, 0.005542755126953125, -0.100341796875, 0.64404296875, -0.78955078125, 1.1572265625, 1.34375, -0.80908203125, 0.97900390625, 0.223876953125, 0.95703125, 0.43603515625, 0.533203125, -2.400390625, 0.73828125, -0.779296875, -0.689453125, 0.90625, 0.1217041015625, -1.33203125, 0.07513427734375, 0.22607421875, -0.5068359375, -0.2333984375, -1.7119140625, 0.049774169921875, 0.133544921875, -0.104248046875, 0.1612548828125, 0.461181640625, -0.465576171875, 0.5498046875, -0.98388671875, 0.65087890625, -0.1971435546875, 1.388671875, 1.9189453125, 0.77001953125, -0.50439453125, 0.307373046875, -0.85693359375, 0.0914306640625, 2.875, 1.05859375, 0.59814453125, -0.337158203125, 0.57470703125, -0.6474609375, 0.425048828125, 0.1185302734375, 0.46435546875, 0.362548828125, 0.41259765625, 0.284423828125, -0.0340576171875, 0.53076171875, 1.6171875, -0.79443359375, -0.697265625, -0.72607421875, 0.1529541015625, 1.3544921875, -0.43994140625, -0.75341796875, -0.7421875, 0.09002685546875, -0.1002197265625, -0.12432861328125, 1.21484375, 1.3408203125, 0.88916015625, -0.625, 0.373779296875, 0.58154296875, 0.34375, -0.022613525390625, -0.03851318359375, 1.408203125, 0.66748046875, 0.72509765625, -1.1904296875, -0.479736328125, 0.69091796875, 0.5166015625, -2.2890625, -0.64599609375, 0.349609375, 0.560546875, -1.072265625, -0.2265625, -0.14306640625, 2.25390625, 0.69873046875, -1.4853515625, 1.57421875, -0.50537109375, 1.357421875, -1.3701171875, -1.3251953125, 0.0750732421875, 1.0283203125, 0.09844970703125, 1.34375, -1.4619140625, -2.9609375, -0.9775390625, -0.123779296875, 1.2939453125, 0.77978515625, 0.403076171875, -1.0107421875, -0.59326171875, -0.73974609375, 0.6455078125, 1.015625, 0.65673828125, -0.04083251953125, -1.2236328125, 0.6044921875, -0.2236328125, -0.20166015625, 0.53759765625, -0.33642578125, -0.6064453125, -0.57080078125, 1.013671875, 1.2568359375, 0.62548828125, 0.64013671875, 1.3076171875, 0.06085205078125, -1.0771484375, 0.2177734375, -0.861328125, 1.3056640625, -1.24609375, -0.294189453125, -0.1396484375, -1.1875, -0.6162109375, 0.1795654296875, 0.2459716796875, 0.279296875, -0.478271484375, -0.833984375, 0.84423828125, 1.40234375, -0.1566162109375, -0.40771484375, -1.2021484375, -0.420166015625, -2.1484375, 0.78955078125, -0.496337890625, -1.150390625, -1.875, -0.09161376953125, -0.9892578125, 0.15185546875, 1.6875, 0.2039794921875, -0.06610107421875, -0.344482421875, 0.49853515625, -0.0167236328125, 0.0112762451171875, -1.345703125, 0.6748046875, -0.73486328125, -2.17578125, -0.05987548828125, -0.71337890625, -0.77490234375, -0.0237884521484375, -1.0732421875, 0.6455078125, -0.89892578125, -0.040008544921875, -0.1290283203125, 0.375244140625, 0.501953125, 0.90966796875, 0.43701171875, 0.6015625, -2.169921875, -0.09185791015625, 1.2109375, -1.6728515625, -1.1982421875, -1.46484375, 1.4765625, 0.7265625, 1.2734375, -0.76806640625, 0.55517578125, -0.11737060546875, -0.040008544921875, 0.76513671875, -1.1142578125, 0.128173828125, -0.9599609375, -0.5478515625, 1.3349609375, -1.71484375, -2.80078125, -0.158203125, 0.56005859375, -0.68994140625, 1.33203125, 0.71044921875, 0.450927734375, -0.327880859375, -0.048187255859375, -1.6640625, -0.36328125, -0.68701171875, -1.052734375, -0.60888671875, -0.144287109375, -1.75, 1.1513671875, 0.1890869140625, -0.149169921875, -1.416015625, 0.736328125, -1.5390625, -0.51513671875, 0.388916015625, -0.93798828125, 0.006992340087890625, -1.3408203125, 0.7509765625, -0.80029296875, -0.9375, -0.61669921875, 0.00121307373046875, 0.423583984375, 0.09539794921875, -0.1639404296875, 0.047027587890625, -1.0908203125, 1.1845703125, -0.814453125, 0.7919921875, -0.6015625, 0.83837890625, -1.2587890625, 0.372802734375, -2.025390625, -0.166015625, -1.6474609375, 1.666015625, 0.82275390625, 0.7041015625, -0.08740234375, -0.59521484375, 0.1884765625, 1.25, -0.1158447265625, -0.49951171875, -1.328125, 0.466552734375, 1.00390625, -2.03125, 0.54150390625, -0.880859375, -0.270263671875, -0.77685546875, 0.759765625, -1.6337890625, -0.587890625, -0.54541015625, -1.2138671875, 0.1820068359375, 0.268310546875, 1.2734375, 0.505859375, -0.53515625, 0.74462890625, 1.3271484375, 0.66015625, 0.39208984375, 3.046875, 0.92578125, 0.77001953125, -0.88525390625, -1.5634765625, 0.85986328125, 0.39453125, -0.416015625, 0.2325439453125, -0.64208984375, 1.755859375, -1.087890625, -1.810546875, -0.77001953125, -0.328369140625, -1.85546875, -1.345703125, -1.34765625, 0.66552734375, -0.4599609375, 0.0374755859375, -1.19921875, -0.304443359375, -0.47802734375, 0.427001953125, -0.927734375, -1.1376953125, -0.92138671875, 0.01364898681640625, 1.521484375, 0.04168701171875, -2.361328125, -0.257568359375, -0.64697265625, -2.67578125, -1.7705078125, 0.37158203125, -0.2315673828125, -0.2469482421875, 0.732421875, -1.4169921875, 2.09765625, 0.5009765625, -0.912109375, -1.306640625, -0.56689453125, 1.267578125, -0.81787109375, 0.366455078125, -1.2900390625, -0.370849609375, 0.09075927734375, -0.86328125, -2.861328125, 0.1617431640625, 0.9580078125, -0.194580078125, -0.64794921875, 1.3994140625, -0.0261688232421875, -0.79150390625, 0.165283203125, 0.296630859375, 0.5791015625, -0.21435546875, 0.65478515625, 0.9775390625, -0.7099609375, 1.61328125, -0.412353515625, -0.1856689453125, 0.294921875, 1.01953125, -0.7578125, -1.0146484375, -0.8125, 2.39453125, 2.974609375, -0.197265625, -1.244140625, 0.03741455078125, 0.1624755859375, 0.379150390625, -0.6064453125, -0.216064453125, -0.81689453125, 1.296875, -0.31689453125, -0.66357421875, 0.92626953125, -2.6875, 0.0830078125, 1.3359375, 0.484130859375, 0.958984375, 0.7265625, -0.021942138671875, 0.56982421875, -1.9140625, -1.3994140625, 0.99462890625, -0.02960205078125, 0.32470703125, 0.94482421875, 0.5283203125, 0.07623291015625, -0.435791015625, -0.327392578125, -0.415283203125, 0.5810546875, -0.003204345703125, -1.80859375, 1.431640625, -0.09503173828125, 1.716796875, -0.1422119140625, -0.29345703125, 0.45849609375, 0.47509765625, -0.7880859375, -0.93115234375, 0.9091796875, -1.0009765625, -0.288330078125, -0.315673828125, -0.044219970703125, 0.1358642578125, -0.8251953125, 0.89013671875, 1.9326171875, 1.2607421875, -0.64892578125, 0.08056640625, 0.01158905029296875, -0.86181640625, 0.77685546875, 0.136474609375, -1.873046875, -1.1025390625, -0.421142578125, -0.62744140625, 0.368896484375, -1.87890625, 0.7646484375, 0.31640625, -0.07861328125, -0.55908203125, 1.0458984375, 1.017578125, -0.96435546875, 0.9833984375, 0.2666015625, 1.6640625, -1.3720703125, -0.7275390625, -2.537109375, 1.0556640625, 1.4326171875, -1.4228515625, 0.0245819091796875, -0.599609375, 0.365478515625, -0.3232421875, 0.30419921875, -0.94140625, -0.99072265625, 0.83935546875, 0.92578125, -0.1875, 0.427490234375, 0.62744140625, -0.185546875, 1.361328125, -0.72265625, 0.49609375, 1.76953125, -0.417724609375, 1.095703125, 2.505859375, -1.91015625, -0.06622314453125, -0.1026611328125, 0.6748046875, 2.8515625, 0.90625, -1.3935546875, 0.326171875, -1.572265625, 0.8994140625, -0.92138671875, -0.260986328125, 0.32958984375, 0.9521484375, 0.253173828125, 0.268310546875, 1.53515625, 0.303955078125, 0.226318359375, 0.54248046875, -0.11883544921875, 0.72607421875, -0.78076171875, 0.1571044921875, -0.26123046875, 0.02001953125, 2.712890625, 0.36376953125, -1.28125, 0.1492919921875, -0.383056640625, -1.6240234375, -1.58984375, 0.5380859375, 0.474853515625, 1.6943359375, 0.8154296875, -0.017181396484375, -1.0517578125, 0.68994140625, -0.2705078125, 0.0360107421875, -0.15283203125, 0.3974609375, -1.7490234375, -0.513671875, -0.62744140625, 1.076171875, -1.9912109375, 1.0859375, -2.056640625, 0.11212158203125, 0.338623046875, 1.078125, -0.00849151611328125, -1.1611328125, -2.083984375, -1.5771484375, 0.81103515625, -0.473876953125, 1.2490234375, -0.10968017578125, 1.048828125, -0.978515625, 0.0736083984375, -1.2158203125, 1.2587890625, 0.70751953125, 0.86962890625, -1.615234375, 0.1414794921875, 1.796875, -0.95654296875, 0.030517578125, 1.0263671875, 1.3701171875, -1.138671875, -0.65869140625, -0.1851806640625, 0.89892578125, -0.62548828125, -1.771484375, 1.298828125, -0.054901123046875, 1.1337890625, -1.5712890625, 0.0433349609375, 0.043304443359375, -0.163818359375, -0.426025390625, -0.2498779296875, -1.9384765625, 1.2080078125, 0.90087890625, -1.30859375, 0.102294921875, 2.021484375, -0.175048828125, -0.374755859375, 0.195556640625, 0.12176513671875, -1.8115234375, -2.421875, -0.101806640625, 0.1298828125, 1.318359375, 0.043487548828125, 1.734375, 3.158203125, 1.755859375, 1.99609375, -1.6044921875, -0.027008056640625, 0.75537109375, 0.9326171875, 0.435546875, 0.521484375, -1.328125, 0.0662841796875, 0.454345703125, -0.2476806640625, -0.95458984375, -0.85888671875, 2.94921875, -0.476806640625, -0.3291015625, 0.352783203125, -1.0849609375, 0.68994140625, -1.9541015625, -1.6533203125, -0.703125, 1.0615234375, 0.419921875, -0.63525390625, 1.0078125, -1.978515625, 0.97802734375, 2.189453125, 0.03631591796875, 1.0400390625, 0.278564453125, -0.49853515625, -0.427490234375, -0.4794921875, 0.83837890625, -2.107421875, 1.2529296875, 0.1903076171875, 1.8056640625, 0.40576171875, -2.01171875, 0.57666015625, -1.9140625, -0.017852783203125, 0.35888671875, 0.197998046875, 0.81103515625, -0.10064697265625, 1.5556640625, -0.75048828125, 0.04608154296875, 1.98828125, 0.7236328125, -0.0633544921875, 0.3623046875, 0.032928466796875, 1.08203125, -0.02789306640625, -0.69482421875, -0.548828125, 0.7578125, -1.982421875, 1.435546875, 1.0068359375, -1.212890625, 0.125244140625, -1.255859375, -0.9814453125, 0.0455322265625, 0.748046875, 1.0712890625, -0.149658203125, -0.7763671875, 0.41162109375, -0.327392578125, -0.5107421875, -1.4052734375, -1.076171875, 0.232177734375, 1.5625, -1.16796875, 0.7041015625, 1.3427734375, -1.33203125, -0.6171875, 0.0233917236328125, -1.4306640625, 0.68603515625, 1.134765625, -0.69384765625, -0.85546875, 1.109375, -0.92529296875, -1.7802734375, 1.00390625, -2.375, -0.181640625, -0.493896484375, 2.259765625, 2.103515625, -0.01251983642578125, 1.025390625, 1.9521484375, 1.583984375, 0.4404296875, 0.1561279296875, 0.481689453125, 0.767578125, -0.71533203125, -0.414794921875, 0.2119140625, -0.267333984375, 1.3935546875, 0.591796875, -0.6767578125, -0.3369140625, -0.80322265625, -1.4560546875, 0.9814453125, -0.50244140625, 0.171142578125, 1.701171875, -0.72265625, 0.272705078125, -0.01436614990234375, 1.6396484375, 0.1728515625, 0.195068359375, -0.006328582763671875, -0.48828125, 0.58837890625, -0.828125, 0.71826171875, -1.4599609375, 0.465576171875, -0.00852203369140625, -0.2442626953125, -1.1884765625, 0.6484375, -1.857421875, 0.640625, 2.65234375, 0.31298828125, -0.497802734375, 1.306640625, -1.50390625, -0.40185546875, -1.0556640625, 0.9873046875, -0.262939453125, 0.19921875, 0.03277587890625, -0.77392578125, 0.1710205078125, 1.1494140625, 0.96533203125, -0.1591796875, 0.20556640625, -0.7626953125, 0.75537109375, -0.1890869140625, -1.3037109375, 1.318359375, 0.77392578125, 0.053802490234375, -0.90869140625, 0.62353515625, 1.4970703125, -0.513671875, 0.63134765625, -0.14794921875, -0.09649658203125, -1.908203125, -0.861328125, -0.1593017578125, -0.347900390625, -0.14990234375, -1.3623046875, -0.286376953125, 0.02276611328125, -0.59130859375, -0.63330078125, -1.7392578125, -0.421875, 1.10546875, -1.5517578125, 2.13671875, 0.4130859375, 1.4892578125, -0.304931640625, 1.23828125, -0.20458984375, -0.501953125, 0.1494140625, 0.1947021484375, 0.27294921875, -0.791015625, -1.21875, -0.67578125, -0.740234375, -0.525390625, 0.7060546875, 1.640625, -1.59375, -0.086181640625, 1.689453125, 0.59716796875, 1.1376953125, 0.82177734375, 0.39892578125, -0.1513671875, 0.2484130859375, 1.189453125, 0.179931640625, 0.56689453125, 1.0419921875, -1.2724609375, -1.6318359375, 0.572265625, -0.51708984375, -1.0537109375, -1.12109375, 0.63623046875, 0.87939453125, 0.95458984375, 0.309326171875, -0.630859375, 0.26171875, -0.9873046875, -0.3515625, 1.1669921875, 2.01953125, -1.23828125, 0.294921875, -0.322265625, -1.7646484375, 0.734375, -0.407470703125, -0.10693359375, 1.1982421875, -1.1669921875, -0.27099609375, -0.20166015625, 0.290771484375, -0.67431640625, -0.93359375, 0.275634765625, -1.1396484375, -2.0078125, 0.46240234375, -3.232421875, 1.5224609375, 1.0732421875, -1.25390625, 0.98291015625, 1.23046875, -1.0703125, 1.9296875, -0.83154296875, -1.6357421875, -0.24365234375, 0.6376953125, 0.63916015625, -0.7998046875, -0.46923828125, -0.65576171875, -0.1441650390625, 1.1328125, 1.0234375, -0.406005859375, 0.3466796875, 0.3076171875, -0.5869140625, 0.69384765625, 1.4794921875, 1.0322265625, 0.87939453125, -0.95458984375, -0.7138671875, 0.66162109375, -0.1463623046875, 0.059722900390625, -0.76171875, 0.096435546875, -0.31982421875, 0.9921875, -1.169921875, 1.31640625, 0.27392578125, 1.17578125, -0.2213134765625, 0.76904296875, -0.051666259765625, -2.19921875, 1.080078125, 1.6201171875, 0.2327880859375, -0.53759765625, -0.3955078125, 0.1812744140625, 0.72412109375, -0.0154266357421875, -2.28515625, -0.54248046875, 1.1728515625, 0.7900390625, 2.173828125, -0.8251953125, -1.21484375, 0.1905517578125, -1.81640625, 0.525390625, -0.771484375, 0.289794921875, 0.328857421875, 1.1552734375, 2.78125, -1.4130859375, 0.377685546875, -0.54833984375, -0.09765625, -0.57666015625, -1.3828125, -0.4326171875, 0.17822265625, 1.107421875, -0.73291015625, 2.068359375, 0.88232421875, 0.86376953125, -1.74609375, 0.56103515625, 0.63525390625, 0.118896484375, 2.220703125, -0.701171875, 0.14599609375, -0.56396484375, -1.5068359375, 1.515625, -0.9970703125, -0.85205078125, -0.8212890625, -0.64501953125, -0.1329345703125, -0.399169921875, -0.95166015625, 0.4912109375, -0.51025390625, -2.021484375, 0.58837890625, -0.07232666015625, 0.595703125, -0.92333984375, -0.1707763671875, -0.271728515625, 0.330078125, -0.200439453125, -0.01226043701171875, 0.010589599609375, 0.806640625, 0.03289794921875, 0.37646484375, 1.4697265625, -0.06427001953125, -0.4697265625, -0.63818359375, -1.0380859375, -0.5830078125, -0.420166015625, -0.410888671875, -1.173828125, -0.259765625, 1.1689453125, 1.0205078125, 0.32080078125, -0.88671875, -0.49951171875, -0.72412109375, -0.92822265625, -0.69140625, 0.52734375, -1.001953125, 0.583984375, -0.9091796875, -0.95458984375, 0.716796875, -1.296875, -0.0222930908203125, -0.175048828125, 0.006786346435546875, -0.81103515625, 2.02734375, 0.091552734375, 0.04876708984375, 1.40625, -0.8896484375, -0.365478515625, 0.98193359375, -1.5263671875, -0.22607421875, 0.927734375, -0.10870361328125, -1.1357421875, -0.312744140625, -0.72021484375, 0.1134033203125, -0.4248046875, -0.7783203125, -0.44140625, -0.89208984375, 1.8759765625, 1.7041015625, 0.06707763671875, 0.44775390625, -1.576171875, 0.71923828125, 0.487548828125, -0.42333984375, -0.33447265625, -1.1123046875, 1.416015625, -0.017974853515625, 1.443359375, 0.90380859375, 1.572265625, 0.54638671875, 0.9013671875, 1.2275390625, -1.96875, -0.32568359375, 0.466552734375, 0.408447265625, -0.09674072265625, -1.9873046875, 0.69287109375, -0.28759765625, -0.279541015625, 0.994140625, -1.1357421875, -0.50927734375, -1.3876953125, 0.136962890625, 0.5751953125, 1.8095703125, -0.190673828125, -1.1083984375, 0.444091796875, -1.08203125, 0.47998046875, 0.342529296875, -1.083984375, -0.1453857421875, 0.5791015625, -1.7373046875, 1.494140625, 0.53076171875, 0.23193359375, -2.23046875, 1.185546875, 1.20703125, -0.59033203125, -0.6416015625, -0.6484375, 0.76513671875, -0.56982421875, -0.310791015625, 0.96435546875, 0.92626953125, -0.2279052734375, -0.1981201171875, -1.373046875, -0.1356201171875, 0.295654296875, -1.2333984375, -0.342041015625, -0.3212890625, 0.439697265625, 0.414794921875, -0.0601806640625, 0.8154296875, 0.64111328125, 1.5478515625, 0.9169921875, -1.25390625, 0.215087890625, 0.054595947265625, 0.00487518310546875, -0.53564453125, -0.474609375, 0.09185791015625, -0.28076171875, -0.92138671875, 2.65625, -0.01493072509765625, 1.947265625, -1.6689453125, -0.495849609375, 0.427734375, -0.64599609375, -0.65771484375, 0.389404296875, 0.3505859375, 2.212890625, 1.3125, -0.281982421875, 1.4482421875, 0.787109375, -0.28955078125, -1.8671875, 0.693359375, -1.3095703125, 0.1795654296875, -1.095703125, -0.6982421875, -1.23828125, -0.4296875, 1.09765625, 1.212890625, 1.341796875, 1.1337890625, 0.828125, -0.7998046875, -0.8740234375, 0.96044921875, -0.28955078125, -1.0029296875, -0.63916015625, 0.033935546875, -0.468994140625, -0.239013671875, 0.78662109375, -0.306396484375, -0.5390625, -0.75439453125, -0.64013671875, -0.76904296875, -1.8212890625, 0.6337890625, -0.322265625, -0.1888427734375, -1.0830078125, 1.3154296875, 0.6015625, -1.078125, -1.3017578125, -1.0400390625, -0.5322265625, 0.80322265625, 1.46484375, -0.58349609375, -0.325439453125, 1.4609375, -0.63232421875, -0.26318359375, -0.044525146484375, 0.186279296875, 2.072265625, -0.205810546875, 0.31103515625, 0.6259765625, 0.347900390625, -0.79443359375, -0.423095703125, -1.2109375, -0.453369140625, 1.4169921875, -1.5048828125, 1.576171875, -0.0165863037109375, 0.93505859375, 0.1878662109375, -0.54638671875, -1.5849609375, -0.71875, 1.28515625, 0.73876953125, -1.994140625, 0.294677734375, -0.024322509765625, -0.0164794921875, 1.2431640625, -0.349853515625, 1.2451171875, -1.169921875, -1.59765625, -0.445068359375, 1.8173828125, 1.5107421875, 0.91552734375, 0.21630859375, 1.1123046875, -1.6484375, 1.4150390625, -0.9462890625, 0.1607666015625, -0.339111328125, 0.04443359375, -1.6865234375, 0.2457275390625, 0.3818359375, 0.4443359375, 0.12249755859375, 1.46484375, -1.57421875, -0.75634765625, -0.32470703125, -2.828125, 0.316650390625, 0.290283203125, -0.92626953125, 1.2509765625, 1.115234375, -0.323974609375, -0.4296875, 0.8740234375, -1.2138671875, 1.96875, -0.12078857421875, -0.78515625, 0.65234375, 0.94580078125, 0.591796875, 1.767578125, -1.1826171875, -1.640625, 0.51220703125, -0.464599609375, 0.50341796875, -0.31591796875, 0.9921875, -0.9638671875, -1.123046875, 0.059417724609375, 0.517578125, 1.6435546875, 0.6171875, 0.9033203125, 1.017578125, 0.1334228515625, 0.388916015625, -0.63818359375, -1.8115234375, 0.3037109375, -0.1376953125, -1.6171875, -0.3564453125, 0.4658203125, 1.4755859375, -0.546875, -0.529296875, -0.6416015625, -0.650390625, 0.4609375, 0.1492919921875, -0.74755859375, 1.7705078125, 1.978515625, -0.76318359375, 1.7099609375, -0.2744140625, 0.96826171875, 0.11083984375, -0.78271484375, -1.1640625, 0.66943359375, 1.318359375, -0.6865234375, 0.62060546875, 0.17431640625, 1.6708984375, 0.23193359375, 0.371826171875, 0.09747314453125, -1.5693359375, 2.193359375, -1.318359375, -1.25390625, -0.08612060546875, -1.1474609375, -0.36962890625, -0.4052734375, -0.2406005859375, -1.0419921875, 0.56640625, -0.42138671875, -2.041015625, -0.84619140625, -0.24462890625, -0.38427734375, 1.1015625, 2.203125, 0.33447265625, -0.3623046875, 0.8203125, 1.2490234375, -0.21142578125, 0.4365234375, 0.8056640625, 0.5615234375, 0.98681640625, -0.09405517578125, -1.9306640625, 0.75390625, 0.264892578125, -1.1171875, -0.806640625, 1.68359375, 0.53271484375, -1.08203125, -0.027923583984375, 0.7607421875, -0.0616455078125, -0.72314453125, -0.276123046875, -1.0771484375, 1.9326171875, 0.1259765625, -1.5986328125, 0.7890625, -0.2291259765625, 0.265380859375, 0.33203125, 0.413818359375, -0.91162109375, 0.34619140625, -0.080078125, -0.67041015625, 0.17822265625, 1.083984375, -1.1669921875, -0.36181640625, -0.120849609375, 0.4404296875, -0.164306640625, 0.7978515625, -0.306396484375, -0.055755615234375, -1.36328125, -0.60009765625, -0.387939453125, -0.417236328125, 0.45556640625, -1.525390625, 0.13427734375, 0.09478759765625, 0.9404296875, 1.1748046875, 0.298583984375, 0.2069091796875, 0.52490234375, -0.505859375, 0.7236328125, -0.212646484375, 0.7314453125, -0.462646484375, 0.27587890625, -0.1597900390625, 1.58984375, 1.0615234375, 0.68798828125, 0.73876953125, -2.076171875, -2.275390625, -0.474609375, 0.814453125, 1.21484375, 0.399169921875, 2.0078125, 0.52587890625, -0.39453125, -0.36181640625, -0.07574462890625, 0.7861328125, -0.394775390625, -0.425537109375, 0.1998291015625, -0.7783203125, 0.90966796875, -0.39404296875, 1.49609375, -1.0830078125, 0.70556640625, 1.0048828125, -0.180908203125, -2.765625, 0.13623046875, 1.30078125, -1.5546875, 1.82421875, 1.8134765625, -1.4052734375, 2.185546875, -0.12744140625, 0.03369140625, -0.751953125, 0.58740234375, -0.56884765625, 0.08697509765625, 0.271728515625, 0.2998046875, 0.1417236328125, 0.400390625, 1.525390625, 2.16796875, -1.9091796875, -3.310546875, -0.57080078125, 0.6201171875, -0.83251953125, -0.037109375, -0.5966796875, -0.5, 0.70263671875, -0.376220703125, 0.7568359375, -0.40234375, 0.357177734375, 0.221923828125, 0.9443359375, 0.77490234375, 0.96630859375, -0.8408203125, -1.0087890625, -0.406494140625, -1.1298828125, 0.454345703125, -1.287109375, -1.4853515625, -0.83642578125, 0.44970703125, 0.08514404296875, -0.62255859375, 0.45166015625, -0.2069091796875, -0.118896484375, -1.6025390625, 0.47607421875, 1.0615234375, -0.38134765625, 1.6630859375, 0.705078125, 0.669921875, -0.448974609375, 0.326904296875, 0.85205078125, 0.5888671875, -0.6806640625, -0.46484375, -1.7236328125, 0.06011962890625, -0.0113983154296875, 1.380859375, -1.47265625, 1.1708984375, 0.206298828125, -0.86474609375, 0.58544921875, -1.142578125, 0.64794921875, -0.91943359375, 1.7841796875, 1.587890625, -0.93310546875, 0.8505859375, -2.076171875, -3.986328125, 0.0097503662109375, 0.434814453125, -1.228515625, -0.7177734375, -0.132080078125, 1.7001953125, -0.051788330078125, -1.9794921875, 0.228515625, -0.49169921875, 1.1533203125, 0.69287109375, -1.6474609375, -1.8857421875, 1.3798828125, -0.71240234375, -1.2890625, 0.78857421875, -0.497314453125, 1.2802734375, 0.2060546875, -0.58837890625, 0.32421875, 1.341796875, 0.55419921875, -1.36328125, -0.422607421875, 0.299072265625, -0.66650390625, -2.1015625, 0.2100830078125, -1.2021484375, -0.60400390625, -1.87890625, -0.093017578125, -0.1417236328125, 1.19921875, 0.09759521484375, -0.34814453125, 0.16064453125, -0.042083740234375, 0.1229248046875, 1.9755859375, -1.8251953125, -0.1845703125, 0.08746337890625, -0.9765625, 1.42578125, 1.7880859375, -0.72509765625, 0.0936279296875, -0.71630859375, 0.2027587890625, 0.6494140625, 0.38525390625, 0.4990234375, 0.947265625, 0.8349609375, 0.451171875, 0.12841796875, -1.974609375, -0.3525390625, 0.6650390625, -0.273681640625, 1.64453125, -0.1856689453125, 0.77734375, -0.7666015625, -0.40966796875, 0.60888671875, 0.77001953125, -0.73681640625, -0.11358642578125, -0.87060546875, 1.03515625, 0.77685546875, -1.189453125, 0.422607421875, 1.65625, -0.275146484375, -0.5361328125, -2.125, 0.279052734375, -1.0712890625, 1.0966796875, -0.256103515625, 1.4404296875, -0.61376953125, 0.8671875, -0.0232696533203125, 0.043701171875, -2.78125, 1.75390625, -0.54541015625, -0.436767578125, 0.27294921875, 0.236083984375, -1.9169921875, -0.1011962890625, -0.16552734375, -2.2890625, 0.312744140625, 0.04205322265625, 0.39892578125, 1.7744140625, 0.5556640625, 0.62841796875, 0.39892578125, 0.95947265625, 0.2188720703125, -1.1064453125, -0.64501953125, 1.095703125, 1.751953125, -0.9052734375, -0.0687255859375, 0.1036376953125, -0.47998046875, -0.0743408203125, -0.260498046875, -0.296875, 0.1971435546875, 0.119140625, 0.70361328125, 0.050933837890625, 2.357421875, 0.1319580078125, 0.0885009765625, -0.456787109375, 0.7763671875, 1.9716796875, -1.048828125, -0.029998779296875, -0.0701904296875, -0.1292724609375, -0.7001953125, -0.97021484375, 0.7607421875, 0.259033203125, 1.2841796875, 0.5810546875, -1.3984375, -0.1531982421875, -0.028839111328125, -0.60546875, 0.99951171875, -1.2744140625, 0.1796875, -0.410888671875, -0.087158203125, 0.61279296875, -1.1962890625, -0.2122802734375, -0.4365234375, -0.68896484375, 0.06646728515625, -1.5673828125, 0.80224609375, -0.845703125, -0.18798828125, 0.82958984375, -0.93310546875, -0.77734375, 0.89697265625, 0.132080078125, -0.76904296875, 0.93115234375, 0.64501953125, -0.347900390625, -0.66796875, -1.4560546875, 1.0185546875, 2.263671875, -0.42236328125, 0.1409912109375, 0.481201171875, -1.8359375, 0.50732421875, 0.8125, 0.64111328125, 0.404052734375, 0.76953125, -0.170654296875, -0.283447265625, 0.86865234375, -0.96923828125, 0.63525390625, -0.50439453125, 0.0038509368896484375, -0.202392578125, 0.86328125, 0.334228515625, 0.469482421875, 0.230224609375, 0.6044921875, 1.1435546875, -1.8232421875, -0.1419677734375, -0.5546875, -1.1708984375, 1.345703125, 0.339599609375, 0.3310546875, 0.70556640625, 0.60693359375, -0.260009765625, -0.6103515625, 1.2578125, -0.231689453125, 1.7177734375, 0.9365234375, 0.397705078125, -0.93798828125, -1.6796875, -1.498046875, 1.3720703125, -0.82080078125, 2.29296875, 1.140625, 0.9921875, -1.5947265625, 2.052734375, -1.7607421875, -1.4833984375, -1.8486328125, -1.3037109375, -0.358154296875, 0.09197998046875, -0.7138671875, -1.1845703125, -2.037109375, 0.70556640625, -0.6015625, 0.76416015625, 0.3056640625, 0.0015869140625, 0.8017578125, -1.7255859375, -0.14990234375, -1.2783203125, 0.5234375, 0.63037109375, 0.818359375, 1.0810546875, -1.2197265625, 1.490234375, -0.51513671875, 0.085693359375, 1.03125, 1.1015625, -0.53955078125, -0.045257568359375, -1.7861328125, 0.1595458984375, 1.3701171875, -1.5478515625, 0.849609375, -0.301025390625, 1.0029296875, -0.07391357421875, -0.61181640625, -0.634765625, 0.0963134765625, 0.354248046875, -0.64892578125, -0.6416015625, 0.282470703125, 1.10546875, 1.9306640625, 1.291015625, -1.1416015625, 0.3720703125, 1.337890625, 0.6005859375, -0.92138671875, -1.8369140625, -0.73046875, 2.291015625, 0.414306640625, 1.4638671875, -0.052978515625, -0.64111328125, -1.3857421875, 0.66796875, 0.07696533203125, 0.105224609375, 0.351318359375, -0.80029296875, -0.041473388671875, 1.4228515625, 0.346435546875, 1.958984375, 0.279541015625, 0.19580078125, 0.8662109375, -0.9423828125, 0.10162353515625, 0.1416015625, 0.51416015625, 0.7724609375, 0.0863037109375, 1.0439453125, 1.3642578125, -1.26171875, -0.50048828125, 0.443359375, 0.40380859375, 0.638671875, 0.2484130859375, 1.30078125, 0.1444091796875, 2.1796875, 0.7216796875, 0.66162109375, -0.9501953125, 0.1529541015625, -1.1044921875, -0.93798828125, -0.66064453125, 0.86083984375, 0.80029296875, 0.79248046875, 1.79296875, 2.603515625, -0.8154296875, 0.1763916015625, -0.41455078125, 1.6943359375, 2.107421875, 0.032958984375, 0.89794921875, 0.328857421875, -1.1435546875, -0.24951171875, -0.1861572265625, 0.9775390625, -0.385986328125, 0.77978515625, 1.2080078125, -0.441650390625, 0.478271484375, -0.3642578125, -0.8349609375, 2.154296875, 0.08599853515625, -0.215576171875, -0.599609375, 0.275634765625, -0.2020263671875, -1.08984375, 0.68359375, 0.8408203125, -0.720703125, 0.8701171875, 1.15234375, -1.869140625, 0.33837890625, -1.48828125, 0.68798828125, -0.144287109375, -0.7587890625, 2.3125, -0.44287109375, -0.0112152099609375, 0.443115234375, 0.55859375, -1.30078125, 2.755859375, -1.189453125, -1.5068359375, -0.28857421875, 0.34765625, -0.80810546875, 0.73779296875, -0.036956787109375, -1.1328125, 0.564453125, -0.8935546875, 0.76953125, -1.23046875, -0.7158203125, -1.2353515625, -0.30126953125, -0.212158203125, 0.2276611328125, -1.2041015625, -0.70166015625, -1.5654296875, -1.0029296875, -0.0926513671875, -0.1683349609375, 0.83837890625, 1.21875, -0.36962890625, -0.67626953125, -0.8251953125, -1.3232421875, 0.04534912109375, 0.77783203125, -1.8662109375, -1.3701171875, 0.253173828125, 0.18603515625, 1.2783203125, 1.9287109375, -0.34619140625, -1.6650390625, -0.705078125, -0.1976318359375, 0.08642578125, -0.72119140625, 1.3251953125, 0.446533203125, 0.13916015625, -0.11151123046875, -0.4267578125, 1.482421875, -1.177734375, -0.66943359375, 0.1251220703125, 0.0911865234375, 0.420166015625, -1.0146484375, 0.231201171875, -0.3955078125, -0.4873046875, -0.58251953125, 0.263671875, 0.927734375, -0.38525390625, -1.119140625, 0.11517333984375, -1.556640625, 0.3369140625, 0.434814453125, 0.11566162109375, -0.2393798828125, 0.29150390625, 2.25390625, -1.185546875, -1.1279296875, 0.038238525390625, 1.9423828125, -0.70361328125, -0.95703125, -0.3291015625, -0.65478515625, -0.2110595703125, 0.379638671875, -0.199951171875, -0.259033203125, 0.54541015625, 0.849609375, -1.4814453125, -0.10400390625, 0.6259765625, -0.074462890625, -2.380859375, 1.3046875, -0.8330078125, -0.31787109375, 0.368896484375, -2.67578125, -0.485595703125, -0.26953125, 0.63720703125, -0.57421875, 2.318359375, -0.495849609375, -0.46875, -0.51318359375, 0.1629638671875, 0.9501953125, -0.438232421875, -0.9765625, -0.425048828125, -0.66650390625, -0.07354736328125, -2.1796875, -0.1663818359375, 0.15673828125, -0.3955078125, -1.83984375, -1.6962890625, -0.216796875, 0.72314453125, -2.07421875, 1.78125, 0.5078125, -1.0556640625, 1.0439453125, -0.5751953125, 1.470703125, -0.52001953125, 1.322265625, -0.64404296875, 0.1280517578125, -0.362060546875, -1.3115234375, 0.6416015625, -0.67041015625, -0.1649169921875, -0.53564453125, -0.95849609375, 0.472412109375, -2.08984375, -1.5849609375, 1.2119140625, 0.1444091796875, 0.83349609375, -0.63232421875, 1.0849609375, -1.6728515625, 0.317138671875, 1.1171875, 0.810546875, -0.458740234375, -0.3046875, -0.272216796875, -0.25830078125, 1.427734375, 0.018768310546875, -0.047027587890625, -0.491943359375, 0.65478515625, -0.1539306640625, -0.13134765625, 1.0732421875, 0.307373046875, 0.98681640625, 0.73046875, -1.96875, -0.908203125, -0.6494140625, -0.239501953125, -0.97900390625, -0.92822265625, -0.9384765625, -0.10888671875, 0.55419921875, 1.615234375, 0.00872802734375, 0.54345703125, -1.37109375, 1.189453125, -1.169921875, 0.505859375, -0.51220703125, 1.3720703125, -0.482666015625, 2.04296875, 0.2008056640625, 1.927734375, 1.3828125, 0.94677734375, 0.6376953125, 0.294677734375, 1.927734375, -0.70556640625, 1.1875, 2.185546875, -1.98828125, -0.463134765625, 0.359619140625, -0.1727294921875, 0.10205078125, -1.7548828125, -0.66845703125, -0.64404296875, 0.54736328125, 0.123291015625, -1.4130859375, -1.5556640625, 1.0224609375, -0.60498046875, 0.41552734375, -0.83447265625, 0.58740234375, -0.646484375, 1.36328125, 0.433349609375, -0.1185302734375, -1.0625, 0.83154296875, 0.10333251953125, -1.51953125, 0.51708984375, 0.59619140625, -0.763671875, -0.2003173828125, 1.5234375, 0.125244140625, -0.353515625, 1.478515625, 1.31640625, -0.351318359375, 1.0654296875, -0.78466796875, 0.609375, 0.1890869140625, 0.93408203125, -0.032012939453125, 0.42626953125, -0.415283203125, 0.417724609375, -0.5869140625, -0.916015625, 1.2041015625, 1.3076171875, -0.7822265625, -0.5224609375, -0.049407958984375, -0.491455078125, 0.308349609375, 0.501953125, -1.2724609375, 1.5869140625, 1.6171875, 0.89794921875, -0.126220703125, -1.5087890625, 0.1982421875, 1.14453125, 0.53466796875, -1.9482421875, 0.56494140625, 0.17919921875, -0.54638671875, 0.8740234375, 1.7216796875, 0.33984375, 0.2301025390625, 0.90576171875, 0.2783203125, 0.1414794921875, -0.30712890625, 1.3408203125, 0.01395416259765625, -0.6279296875, -0.04833984375, -1.0849609375, 0.91259765625, 0.416259765625, -0.48779296875, -0.032012939453125, 2.2421875, 0.040679931640625, -0.7138671875, 0.298828125, -0.962890625, -1.0537109375, 1.5556640625, 0.6142578125, -0.63525390625, 0.10321044921875, -0.0948486328125, -0.084228515625, -1.16796875, 0.2335205078125, 1.4208984375, -1.462890625, 1.6015625, 1.0439453125, 0.056976318359375, -0.87255859375, -1.2841796875, 0.242919921875, 0.7666015625, -0.77978515625, 1.4404296875, 0.5615234375, -0.58984375, -2.35546875, -0.33447265625, 0.086181640625, 0.71337890625, 0.319091796875, 0.55810546875, 0.41845703125, -0.73291015625, -1.0595703125, 1.0927734375, 0.358642578125, -0.52099609375, -0.452392578125, 0.89453125, 1.1689453125, 0.63818359375, -1.6845703125, 1.14453125, 0.2149658203125, 0.2259521484375, 1.431640625, 1.8779296875, -0.80615234375, -0.0784912109375, -0.1671142578125, 0.19873046875, 0.0599365234375, -0.8564453125, -0.15087890625, -0.4599609375, 0.5625, 0.229736328125, 1.3369140625, 1.0869140625, 0.1851806640625, 0.58642578125, -0.242431640625, 0.436767578125, -1.111328125, 0.0841064453125, 1.4365234375, -1.6572265625, -1.08984375, 0.423828125, 1.4951171875, 1.1171875, -0.81689453125, 0.056610107421875, -0.035614013671875, 0.66259765625, 0.1837158203125, -1.640625, -0.6494140625, 0.64599609375, -0.0994873046875, -1.826171875, -0.410400390625, 0.56396484375, -0.2415771484375, -0.7197265625, -1.001953125, 0.75, -0.7470703125, 0.50390625, -1.0146484375, 0.12347412109375, 2.03515625, -0.448974609375, -1.0146484375, 1.994140625, -0.428955078125, -0.619140625, 0.027587890625, 2.498046875, -0.1156005859375, 1.126953125, 0.83447265625, -0.75, 0.464599609375, -0.286865234375, 0.040283203125, 0.389892578125, -0.2030029296875, -0.48193359375, -0.006778717041015625, 0.33984375, 2.0546875, -0.6767578125, 1.359375, 0.80029296875, 0.7548828125, 1.109375, 1.880859375, -0.0244293212890625, -1.1708984375, 0.541015625, 0.0304107666015625, -2.150390625, -1.7158203125, 1.150390625, -0.245361328125, 2.51171875, 1.73046875, -0.71875, 0.4892578125, 0.89404296875, -0.422119140625, 0.0286407470703125, 0.1424560546875, 0.80419921875, -0.392578125, -0.494140625, -0.2037353515625, 0.79833984375, 1.9443359375, -0.53466796875, -0.69921875, -0.40576171875, 1.4375, 0.75048828125, 0.3251953125, -0.806640625, 0.9443359375, 0.93212890625, -0.85400390625, -1.056640625, -0.1416015625, 0.452880859375, 0.32275390625, -0.2493896484375, -0.66162109375, 0.59912109375, 1.9111328125, -0.49560546875, -1.2890625, 1.6279296875, -1.7216796875, 0.48681640625, 0.82958984375, 1.0751953125, -1.7890625, -0.83642578125, 1.5947265625, 1.384765625, -0.372314453125, -0.17626953125, 1.1923828125, 0.9150390625, -0.0396728515625, -0.5283203125, 2.0390625, -1.3623046875, -0.263427734375, 0.81396484375, -1.4658203125, -1.453125, 0.441650390625, -0.583984375, -0.5126953125, 0.55712890625, 0.8837890625, 0.1771240234375, 0.04864501953125, 0.83642578125, 0.59375, 0.5126953125, -1.5595703125, -0.81396484375, -0.59423828125, 0.625, -1.12109375, -0.3212890625, 1.6103515625, -1.3466796875, -0.005397796630859375, 1.4794921875, 2.140625, 0.84716796875, 0.0069732666015625, -0.1729736328125, -0.70166015625, -2.009765625, -0.42041015625, 2.41796875, -2.044921875, 0.1689453125, -0.43115234375, 0.64404296875, 0.85400390625, 1.7294921875, -1.0361328125, -0.2509765625, 0.92333984375, -1.642578125, 0.0859375, -1.4462890625, -0.2366943359375, 0.67724609375, -0.474609375, -0.87060546875, -1.5205078125, -0.285400390625, -0.1119384765625, 0.73388671875, -1.0576171875, 0.06634521484375, -2.064453125, -0.1197509765625, 0.3994140625, -0.5908203125, -0.93603515625, 0.9873046875, 0.498779296875, 0.458740234375, 0.57568359375, 0.82080078125, -1.6806640625, 0.71923828125, -0.00439453125, 1.5830078125, -0.17529296875, 0.327392578125, 0.810546875, 0.114990234375, -0.31396484375, 0.8203125, -0.8740234375, 0.0882568359375, -0.005802154541015625, 0.57080078125, 0.0697021484375, -0.9365234375, 0.7294921875, -0.30517578125, -0.497314453125, 0.14501953125, -1.0244140625, 1.98046875, -1.46484375, 0.292724609375, 0.047943115234375, -1.390625, 0.11376953125, -0.4912109375, 0.01287078857421875, -1.8544921875, -0.71435546875, 1.373046875, 0.167236328125, 0.256103515625, -0.90869140625, -0.032440185546875, 0.5205078125, -1.40625, 1.693359375, 0.83154296875, -0.026947021484375, -0.740234375, -0.24755859375, -1.2021484375, -1.404296875, 1.6201171875, 0.205078125, -0.089111328125, 1.3310546875, -0.58935546875, -0.90625, 0.64404296875, -0.07861328125, 0.521484375, -0.103271484375, 1.6005859375, -1.9619140625, -0.3291015625, -0.2086181640625, 0.96337890625, 0.18603515625, 1.0888671875, 0.357421875, 0.59326171875, -1.32421875, 0.5693359375, -0.2459716796875, 0.42529296875, -0.259521484375, -0.6591796875, 0.36083984375, -0.27099609375, 0.658203125, -0.8310546875, -1.2001953125, -0.0269622802734375, -0.19140625, -1.439453125, -0.81640625, -2.603515625, -1.2783203125, -0.986328125, 1.646484375, 0.845703125, 1.8408203125, -1.23828125, -0.0311126708984375, -1.3662109375, -1.1044921875, -0.50146484375, 0.298583984375, -0.59130859375, -0.353271484375, 2.58984375, -0.580078125, 1.2421875, 0.276611328125, -0.96923828125, -0.76171875, 1.5517578125, -0.339599609375, 0.409423828125, -0.0362548828125, 1.06640625, 1.2919921875, 0.205322265625, 0.305419921875, -2.087890625, 0.9033203125, -0.376708984375, 0.033660888671875, -0.98388671875, -1.2333984375, 1.40625, 0.7880859375, -0.51513671875, -1.2314453125, -0.245361328125, 0.2215576171875, -1.7490234375, -0.02978515625, 0.17333984375, 0.382080078125, 1.4443359375, -0.96875, 0.462158203125, 0.47412109375, -1.4482421875, -0.52783203125, 2.224609375, 0.09039306640625, -1.2099609375, -2.369140625, -0.5654296875, -0.5166015625, -2.4140625, -0.114501953125, 0.93408203125, 0.2266845703125, -1.1806640625, -0.62548828125, 0.02960205078125, -1.55078125, -0.56884765625, -2.240234375, 1.8388671875, -0.262451171875, 0.91015625, 0.24462890625, -1.193359375, -1.078125, -0.4853515625, 0.0985107421875, 0.705078125, 0.52490234375, -0.72021484375, 0.046539306640625, 0.11444091796875, -0.393798828125, -1.4462890625, -0.7744140625, -0.646484375, 0.376953125, -0.4052734375, 0.3779296875, 0.44873046875, -1.30859375, 2.29296875, -0.29443359375, -0.1470947265625, 0.90283203125, 2.013671875, 1.9541015625, -0.826171875, -1.341796875, 0.241943359375, 1.1884765625, 0.2841796875, 0.70556640625, -0.3427734375, 1.169921875, -0.681640625, 0.131103515625, 0.360107421875, -0.033294677734375, -0.333984375, 1.7724609375, 1.1484375, -0.1676025390625, -0.74853515625, 0.599609375, -0.42041015625, -0.4794921875, -1.736328125, 1.1220703125, 1.59375, -0.2200927734375, -1.314453125, 0.271484375, -0.205810546875, -0.65869140625, -0.4892578125, -0.93310546875, 1.6904296875, 0.414306640625, 0.014373779296875, 0.07281494140625, -1.16015625, 1.8662109375, 0.25537109375, -0.927734375, 1.2470703125, 0.94189453125, -0.8798828125, 0.9677734375, 0.81787109375, -0.51318359375, 2.423828125, -0.1822509765625, 1.2685546875, -1.1376953125, 0.7314453125, 0.477783203125, -1.3232421875, 0.30078125, 0.298583984375, 0.428955078125, 0.86083984375, -0.8408203125, -0.78271484375, -0.006649017333984375, 0.254150390625, -0.5126953125, 1.94140625, -0.91650390625, -0.161865234375, 0.5771484375, -1.5068359375, -0.5283203125, 1.0810546875, -0.619140625, -1.9853515625, -1.4072265625, -1.505859375, 0.401611328125, -0.45703125, -0.399658203125, -0.157470703125, 1.69921875, -1.935546875, 0.10711669921875, -0.99853515625, 2.13671875, 0.157958984375, 0.313720703125, 0.38525390625, 1.1123046875, -0.501953125, 2.607421875, 0.285400390625, -0.306396484375, -0.50439453125, 0.71875, 0.62548828125, 1.126953125, 2.30078125, -0.3857421875, -0.174072265625, 0.9482421875, -0.9169921875, -0.33544921875, -0.45263671875, 0.2919921875, -0.3642578125, -0.2236328125, -0.732421875, 0.2476806640625, 0.307373046875, 0.77099609375, -0.051422119140625, -0.58544921875, -0.80419921875, 0.6572265625, 0.26611328125, -1.037109375, -0.95654296875, -0.1702880859375, 0.3115234375, 0.1251220703125, 0.375, 0.27978515625, -0.1209716796875, -1.5341796875, 0.736328125, 1.0830078125, -0.91552734375, -1.16015625, 2.23046875, 2.037109375, -0.30615234375, 0.8681640625, 0.61865234375, 0.68212890625, 1.4169921875, -0.9189453125, -0.458251953125, -0.4677734375, -0.94873046875, 1.2294921875, -0.2088623046875, -0.59521484375, -0.8349609375, -2.00390625, -0.06842041015625, 0.2174072265625, 1.068359375, -0.0885009765625, 0.71875, -0.529296875, -2.310546875, 1.177734375, -2.251953125, 0.282958984375, 1.1796875, -0.293212890625, 0.267333984375, -0.3798828125, -0.51025390625, -0.26904296875, 1.0546875, 0.04742431640625, -0.666015625, 1.267578125, -0.5693359375, -0.45068359375, 0.2105712890625, 0.58544921875, 0.00673675537109375, 0.640625, 1.7490234375, 0.26806640625, 0.05511474609375, -0.1099853515625, -1.3955078125, -0.76611328125, -0.0814208984375, -0.58837890625, -0.66943359375, -1.1962890625, -0.4326171875, -2.0, -0.70361328125, 1.4873046875, 1.0, -0.81201171875, 0.33740234375, 0.12103271484375, 0.99560546875, 0.896484375, 0.51171875, 0.470703125, -1.6484375, 1.7744140625, 0.8271484375, 0.0133514404296875, -1.3046875, 1.3310546875, -0.52392578125, 0.6865234375, -0.358154296875, -0.341064453125, 0.3955078125, 0.59228515625, 0.5380859375, -0.91259765625, 0.638671875, 0.240966796875, 1.0615234375, 0.9951171875, 0.52001953125, 0.0882568359375, -1.806640625, -0.7568359375, 0.3603515625, -0.34814453125, 0.42333984375, 0.31298828125, 1.2744140625, 0.07354736328125, -0.25048828125, 0.34716796875, 0.1910400390625, -0.54150390625, -0.5703125, -1.84765625, -1.3056640625, -0.2626953125, 1.52734375, -0.332275390625, -1.0927734375, 0.93798828125, -0.261962890625, 1.3369140625, -0.364501953125, -0.074462890625, -1.509765625, -1.052734375, -0.279296875, -1.267578125, -1.443359375, 1.0810546875, 1.763671875, -0.417236328125, 0.9013671875, 0.9140625, -1.2265625, -2.080078125, -1.0078125, 0.32177734375, -1.33984375, -0.6279296875, -1.6728515625, 0.27587890625, 0.57080078125, -0.56298828125, 2.03125, 1.513671875, 0.923828125, -2.455078125, -1.5517578125, -1.6953125, 0.286865234375, 0.923828125, -0.10601806640625, -0.07684326171875, -0.85791015625, 2.005859375, 0.06622314453125, -0.334228515625, 0.048126220703125, 0.368896484375, 1.287109375, -1.0078125, 0.345703125, -0.1646728515625, -0.464599609375, 0.8779296875, 0.196044921875, -1.8115234375, -2.1796875, -0.425048828125, -0.64794921875, 1.8701171875, -0.43603515625, -0.452392578125, -2.033203125, 0.477783203125, 0.365966796875, -0.404052734375, -0.9775390625, 2.587890625, 0.1453857421875, 0.80908203125, -0.96875, -0.66455078125, 0.1607666015625, 0.82421875, 1.6142578125, -1.9306640625, -0.48974609375, 1.1875, 2.13671875, -1.5751953125, -0.421875, 0.67236328125, -0.08880615234375, -0.0941162109375, -2.8984375, 0.26806640625, -0.9208984375, 0.75244140625, 0.0869140625, -0.1845703125, 1.8642578125, 0.1158447265625, -1.34765625, 0.12249755859375, -0.1846923828125, 2.291015625, 0.32421875, -1.3544921875, -0.244384765625, -0.1292724609375, -0.0014095306396484375, 0.007442474365234375, -1.1982421875, -1.2734375, 0.0170440673828125, 0.57275390625, 0.258056640625, 1.6962890625, -0.21435546875, -0.7294921875, -0.46435546875, 1.1318359375, -0.0731201171875, -0.77978515625, 0.350341796875, 0.06463623046875, 0.55078125, 0.5791015625, 0.1590576171875, -0.1322021484375, 0.11175537109375, 1.23828125, 0.233154296875, -0.6533203125, -0.302490234375, 1.904296875, -0.3740234375, 1.818359375, 0.2257080078125, 0.2841796875, 0.6396484375, -0.379150390625, -0.1005859375, -0.015899658203125, 0.490478515625, -0.9609375, 0.36181640625, -0.2978515625, 0.254638671875, -0.430908203125, -0.5205078125, 0.06939697265625, -0.293212890625, -0.385009765625, 0.184814453125, -0.9287109375, 0.73779296875, -0.5615234375, 0.04046630859375, -0.2353515625, 0.75830078125, -0.7158203125, -0.68603515625, 0.50341796875, 0.7578125, -1.3681640625, -0.1300048828125, 0.7978515625, 0.2115478515625, 0.54052734375, -1.54296875, -0.004520416259765625, -0.0254364013671875, 0.62255859375, 1.3818359375, 0.007587432861328125, -0.81640625, -0.17333984375, -1.0380859375, -0.54541015625, -1.1591796875, 0.2734375, -1.7333984375, 1.0126953125, -0.057708740234375, -0.51171875, -0.5712890625, -0.56689453125, 1.0341796875, 0.34130859375, 0.188232421875, 0.7294921875, 0.338623046875, -0.58251953125, -0.341796875, -1.3603515625, -1.697265625, -0.1253662109375, -3.00390625, -0.314697265625, -0.0248565673828125, 0.93701171875, -0.391357421875, -0.93017578125, 0.80126953125, 0.364990234375, 1.8037109375, -1.0703125, -0.55224609375, 1.0791015625, -0.015777587890625, -0.47314453125, 1.861328125, 1.0078125, -0.7919921875, 0.6826171875, 1.640625, 0.95166015625, -0.2191162109375, -1.259765625, 0.77099609375, -0.88427734375, 0.57275390625, -0.6982421875, -1.009765625, -0.1466064453125, -0.7080078125, 1.6513671875, 0.576171875, -0.9755859375, -0.3251953125, 0.6396484375, -2.26171875, -0.701171875, 0.1142578125, -0.295166015625, 0.1297607421875, 0.31396484375, 0.63330078125, -1.0712890625, -0.2470703125, -2.576171875, -0.25, 0.2215576171875, 0.75, 2.076171875, -1.501953125, -2.712890625, 1.7265625, 0.60888671875, -0.55126953125, -0.6337890625, 1.185546875, -0.421630859375, 0.150634765625, -0.21630859375, 0.5576171875, 1.0625, -1.0029296875, 1.037109375, 0.517578125, 1.0625, -0.79345703125, -0.459228515625, 0.1534423828125, -0.0633544921875, 0.6669921875, 1.4267578125, 2.33984375, -2.462890625, 1.0517578125, -0.325439453125, 1.0947265625, 0.1005859375, -0.7265625, -1.1279296875, 0.302978515625, -1.205078125, 0.274169921875, -0.74462890625, -0.10784912109375, 0.55859375, 0.556640625, -1.1865234375, -0.88818359375, 0.75341796875, -1.5400390625, 2.177734375, -0.7373046875, -0.6279296875, 0.70361328125, 0.27197265625, 0.2137451171875, 0.4833984375, -0.312255859375, 0.2266845703125, -1.814453125, -2.546875, -0.42041015625, -0.37890625, 3.232421875, 2.57421875, 1.2392578125, 0.1353759765625, 0.3828125, -0.358154296875, -0.2152099609375, 0.27392578125, 0.47705078125, 0.6533203125, -0.260986328125, -0.239990234375, 0.33251953125, 0.8974609375, 0.2396240234375, 0.623046875, -0.8974609375, -0.30517578125, -0.46142578125, -1.5908203125, -0.60546875, -1.2451171875, 2.080078125, 0.8759765625, -0.26416015625, -1.2080078125, 1.34765625, 0.724609375, 0.72509765625, 0.042449951171875, -1.640625, 0.359130859375, 2.33984375, 2.12109375, -0.52099609375, -0.80029296875, 1.1337890625, 0.250732421875, 1.052734375, -0.26220703125, 1.9853515625, 0.356689453125, -0.56884765625, 1.0048828125, 0.60498046875, -1.0830078125, -2.107421875, -0.8408203125, 0.416748046875, 0.0931396484375, -0.3076171875, -0.052398681640625, -0.9765625, -0.0650634765625, -0.2342529296875, -1.7314453125, -0.8740234375, -1.181640625, -1.1455078125, 0.76708984375, 0.42138671875, -0.6923828125, 0.2269287109375, -0.208740234375, -1.359375, 0.2325439453125, -0.91943359375, 0.61181640625, -0.46044921875, -0.76220703125, 1.0341796875, 1.4501953125, 1.0546875, 1.103515625, 0.63623046875, 0.252685546875, -0.53125, 1.1884765625, -0.5283203125, 1.55859375, 0.0703125, -1.580078125, 0.26708984375, -0.434814453125, 0.56591796875, -0.65576171875, -0.42431640625, 0.405029296875, 0.06207275390625, 2.501953125, 0.5048828125, 0.92236328125, -0.25634765625, 0.97021484375, -0.25537109375, 0.9599609375, 0.408447265625, 0.414306640625, 1.22265625, -0.345947265625, 0.75634765625, -0.197998046875, 0.8232421875, -0.446044921875, 0.9873046875, -2.52734375, 0.65478515625, -0.312255859375, 0.016693115234375, 1.2666015625, -0.1519775390625, -1.564453125, -2.888671875, 0.04302978515625, 1.759765625, 0.2408447265625, 0.459228515625, -0.55419921875, -1.919921875, 0.73046875, 2.123046875, 1.42578125, 0.53662109375, -0.60888671875, -0.38330078125, 2.380859375, -0.11077880859375, 2.38671875, 1.2802734375, -0.1409912109375, 0.83056640625, -0.62109375, 0.576171875, -0.51708984375, 0.6826171875, -0.1334228515625, -0.004116058349609375, -0.4091796875, -1.1962890625, -0.29052734375, -0.1400146484375, -0.6142578125, 0.281005859375, -0.626953125, 0.35595703125, 0.51904296875, -1.3896484375, 0.43798828125, 1.224609375, -1.2109375, -0.96875, -0.1851806640625, 0.64892578125, -2.908203125, -0.89599609375, -1.17578125, 1.328125, 0.58984375, 1.6064453125, -0.391845703125, 0.3154296875, 1.1259765625, -0.78662109375, 0.05743408203125, 0.7900390625, 0.047027587890625, -0.301025390625, 0.5146484375, 0.1361083984375, 0.5419921875, 0.483642578125, 0.548828125, 0.304931640625, -0.460693359375, 1.8115234375, 0.1734619140625, -0.0728759765625, -0.75146484375, 1.01953125, -0.783203125, -0.2479248046875, 1.0458984375, 0.7001953125, -2.01171875, -0.50537109375, 0.370849609375, 0.04315185546875, -1.15234375, 0.037872314453125, 1.12109375, 0.4130859375, 0.66015625, -0.2373046875, 0.197998046875, 0.58154296875, 0.90625, 0.25830078125, 0.019775390625, -0.96435546875, -0.7783203125, 0.5205078125, 0.038726806640625, -0.0716552734375, 0.7294921875, 2.38671875, 0.2103271484375, -0.11724853515625, 0.75634765625, 1.1396484375, 0.07305908203125, -1.5087890625, 0.7080078125, 0.09906005859375, 0.77001953125, 0.36962890625, -1.7529296875, 0.1912841796875, 0.463623046875, 0.25341796875, 2.283203125, -1.4736328125, 0.5185546875, 0.66357421875, 0.37158203125, 0.022430419921875, -0.92529296875, -0.400146484375, -1.43359375, 0.2012939453125, 0.88134765625, -0.7646484375, 0.60888671875, -0.2393798828125, -0.029510498046875, 0.12115478515625, 2.6953125, 2.14453125, 1.15234375, -1.7265625, 0.1947021484375, 0.77294921875, -2.162109375, -1.6884765625, 1.013671875, 0.79150390625, -0.568359375, -0.76318359375, 2.896484375, 0.10772705078125, -0.11004638671875, 0.5986328125, 1.37109375, 0.6513671875, 0.185302734375, -1.1708984375, 0.29296875, 0.2130126953125, -0.20166015625, -2.173828125, 1.2509765625, 0.7880859375, -0.460205078125, -1.9404296875, 1.373046875, -2.3515625, -0.93017578125, 0.22802734375, 0.499267578125, -1.2265625, -0.0101318359375, -1.7373046875, 1.3916015625, -0.49267578125, -1.0556640625, -2.052734375, -1.7724609375, 0.053466796875, -1.00390625, 0.2440185546875, 0.1236572265625, 1.0615234375, -2.310546875, -0.0101776123046875, -0.55517578125, 1.34375, 0.30517578125, -0.96533203125, 1.453125, -1.6494140625, -0.103515625, -0.06329345703125, 0.88232421875, 0.70166015625, 0.465576171875, -0.6611328125, -1.2724609375, 0.372802734375, 1.40625, 1.705078125, 0.63427734375, -0.385498046875, -0.0286865234375, 1.197265625, 0.9697265625, -0.45556640625, 0.921875, -0.0033130645751953125, 0.96728515625, -0.11663818359375, -0.133544921875, -0.0478515625, -0.255615234375, 1.0654296875, 0.169921875, 1.1767578125, 0.04986572265625, 0.1744384765625, -0.5078125, 0.52734375, -0.86328125, -1.5009765625, 0.26953125, 0.495361328125, 0.007366180419921875, 0.603515625, 0.439453125, -0.1927490234375, 1.2724609375, -1.26171875, -1.1396484375, 0.71435546875, 0.97021484375, 0.91259765625, 0.642578125, -1.51171875, 1.6044921875, 0.64794921875, 0.81640625, 0.367919921875, -0.2271728515625, 0.2386474609375, 0.64794921875, -1.2255859375, -1.0419921875, -0.218505859375, -0.76416015625, -0.79296875, 0.499267578125, -0.020172119140625, 0.07806396484375, 1.30859375, -0.07696533203125, 0.345947265625, -1.888671875, 1.3447265625, 0.71630859375, 0.736328125, 0.95703125, -1.921875, 0.81396484375, 1.505859375, -0.12127685546875, -1.5693359375, -0.98046875, -0.257568359375, -0.363525390625, 0.15771484375, -1.701171875, 1.060546875, 0.203125, 0.5029296875, 0.71875, -0.6181640625, 0.8896484375, -0.442138671875, 0.4580078125, -1.611328125, 0.448974609375, 0.40673828125, 0.6689453125, -0.309326171875, 1.115234375, -0.11297607421875, -0.859375, 0.7216796875, 1.3388671875, -0.2490234375, -0.5966796875, 0.31591796875, -0.41650390625, 0.958984375, 1.5263671875, -0.5732421875, 1.140625, 2.033203125, 0.61083984375, 0.814453125, 0.91796875, 1.3525390625, 0.81640625, -0.7841796875, 0.79931640625, -0.296875, 1.8388671875, 1.4326171875, 0.351806640625, 0.048126220703125, 0.333740234375, 0.68359375, 0.324462890625, 2.05859375, 1.5810546875, -0.1573486328125, 0.1845703125, 1.74609375, -1.044921875, 1.1865234375, 0.54052734375, 0.5556640625, -0.5830078125, 1.181640625, -0.4521484375, 0.037109375, 1.0361328125, -1.24609375, -0.380126953125, -0.74072265625, -0.52587890625, 0.00823974609375, 0.252197265625, -1.373046875, 1.0517578125, -0.5751953125, -0.57666015625, 0.65625, -0.97998046875, -0.23046875, 1.2392578125, 1.2861328125, 1.52734375, 1.328125, 0.9677734375, -0.94287109375, 0.0675048828125, 0.32568359375, -0.556640625, -0.44140625, -0.74951171875, 0.95361328125, 0.81640625, 0.048187255859375, 0.02154541015625, -0.468505859375, -1.1083984375, -0.884765625, -0.275390625, 1.0888671875, 1.0556640625, -0.458740234375, 0.951171875, 0.02264404296875, -0.6142578125, -0.291748046875, 0.89892578125, -1.2861328125, 1.181640625, -0.265380859375, 0.48388671875, -0.150146484375, -1.3955078125, -0.79931640625, -0.8125, -1.3759765625, -2.013671875, 1.380859375, -1.5810546875, 0.63134765625, 0.8642578125, -1.279296875, 1.197265625, -0.5888671875, -0.0859375, -1.423828125, 2.66015625, 0.1417236328125, 1.0791015625, 0.1690673828125, -1.5341796875, 1.3603515625, -0.9931640625, 0.09429931640625, -1.794921875, 1.560546875, 0.462646484375, 0.241943359375, -0.0748291015625, 0.6845703125, 0.062103271484375, -1.9599609375, -0.0587158203125, 0.61767578125, 0.6337890625, 0.1597900390625, -0.66650390625, 0.067138671875, -0.08941650390625, -1.8251953125, -0.10845947265625, 1.130859375, 1.0029296875, -1.55078125, 0.55322265625, 0.143310546875, -0.357666015625, -1.9658203125, 0.75537109375, 0.03485107421875, -1.0166015625, -1.14453125, -0.14599609375, 0.0574951171875, -1.509765625, -1.0087890625, -0.5341796875, -0.61865234375, -0.2197265625, 0.7685546875, -1.580078125, 0.515625, 0.0008001327514648438, -0.202880859375, 0.046112060546875, -0.30859375, -0.88427734375, -0.1480712890625, 0.939453125, -0.337646484375, 1.28125, -1.181640625, -2.81640625, 0.61181640625, 0.955078125, -1.7431640625, -0.3779296875, -2.01171875, -0.32666015625, -0.76123046875, -0.66650390625, -0.272216796875, 0.912109375, 0.36083984375, -0.443603515625, -0.1181640625, -1.1103515625, -0.2391357421875, -1.21875, 1.1796875, 1.068359375, -0.94189453125, -0.271484375, 0.08111572265625, -1.4287109375, 0.061553955078125, 1.0322265625, -0.08013916015625, -0.2191162109375, -0.9677734375, 1.134765625, -1.18359375, 1.1591796875, 1.51171875, -0.300048828125, 0.401611328125, -0.32080078125, 1.603515625, -1.01953125, 0.441650390625, 0.85205078125, -0.77001953125, -0.2083740234375, -1.0595703125, 0.5283203125, -0.78857421875, 0.66748046875, -1.6728515625, 0.302734375, 0.50634765625, -1.5556640625, -0.75146484375, 0.529296875, 0.38037109375, -1.16796875, -0.361328125, -0.4560546875, -0.08514404296875, -1.703125, -0.2044677734375, 0.63134765625, -0.5966796875, -0.95458984375, 0.638671875, -1.5537109375, 0.246337890625, 0.53466796875, 0.36474609375, -0.12481689453125, -0.52099609375, 0.99560546875, -0.669921875, 0.5439453125, 1.1064453125, 0.728515625, 0.59228515625, 0.124267578125, 0.29541015625, -1.6455078125, -0.428466796875, -1.3720703125, 1.1474609375, -0.10296630859375, -0.1654052734375, -1.5634765625, 0.0157012939453125, -0.8359375, -0.1248779296875, -0.391845703125, 0.8671875, 1.2236328125, 0.529296875, 1.2763671875, 0.53564453125, -0.97119140625, -3.099609375, 1.296875, 1.498046875, -0.68212890625, -1.5, -0.168701171875, -0.49658203125, 0.771484375, 0.734375, -1.1171875, -1.08203125, -0.80078125, 2.626953125, -0.134521484375, -1.2509765625, 0.71484375, -0.6796875, -0.6982421875, 1.056640625, 0.39990234375, -0.55078125, -1.470703125, 0.2093505859375, 1.025390625, -0.429443359375, 1.1796875, 0.3564453125, -1.3779296875, 1.2001953125, -0.1298828125, 0.232666015625, 1.5791015625, -0.75244140625, 0.0521240234375, -1.5458984375, -1.248046875, 0.80126953125, 0.435302734375, 0.85595703125, -1.7705078125, -1.083984375, -0.96533203125, 0.7861328125, 0.12152099609375, 0.148681640625, 1.3017578125, -0.59326171875, -0.29931640625, 0.060882568359375, 2.01171875, -1.22265625, 0.287353515625, 0.0229949951171875, -0.2392578125, -0.61376953125, 0.90673828125, 0.83935546875, 0.6064453125, -0.0184783935546875, -1.6181640625, 0.318115234375, -0.994140625, -0.266845703125, 0.453369140625, -0.10784912109375, 0.83154296875, -0.84228515625, 0.367919921875, -1.0966796875, 0.8740234375, -0.2332763671875, 1.0537109375, 0.70751953125, -0.72998046875, 0.708984375, -0.07257080078125, -0.25537109375, -0.9619140625, -1.4541015625, 0.8818359375, -0.9306640625, -0.9443359375, 0.43408203125, 1.751953125, -1.6650390625, 0.92626953125, 1.607421875, -0.01226806640625, -0.129638671875, 1.150390625, -0.390625, 0.1092529296875, 1.109375, 0.399658203125, -0.235595703125, -0.01457977294921875, 0.55517578125, -0.338134765625, 0.53466796875, 0.276611328125, -2.4296875, 1.611328125, -1.197265625, -0.5439453125, -0.320556640625, -0.035980224609375, -1.609375, -3.03515625, -0.0274200439453125, 0.0789794921875, -1.0458984375, -0.065185546875, 0.9384765625, -0.355224609375, -1.3486328125, -0.498779296875, 0.253173828125, 0.40185546875, 1.259765625, 0.326416015625, 0.56494140625, 1.1240234375, 0.46728515625, -0.552734375, 0.395751953125, 1.3642578125, -0.994140625, -1.3564453125, -1.1806640625, 0.2027587890625, -0.4697265625, 0.382080078125, 0.11572265625, 0.63134765625, -0.01300811767578125, 0.85693359375, 1.625, -0.70751953125, 0.427734375, 0.1856689453125, 0.44970703125, 0.319580078125, 0.1759033203125, 0.334228515625, 1.5595703125, -1.1904296875, 0.5625, 0.033050537109375, -1.302734375, 1.2734375, -1.5927734375, -0.64892578125, -0.91748046875, 0.2724609375, -1.3583984375, -0.7607421875, -0.2459716796875, -0.2568359375, -0.0143280029296875, 0.11297607421875, -0.36083984375, -1.171875, 1.2998046875, -0.92529296875, 0.97412109375, -0.994140625, -0.7724609375, 1.1484375, 0.0479736328125, -0.1544189453125, -0.455078125, -0.619140625, -1.4658203125, -0.0848388671875, -0.10498046875, 1.7880859375, 0.85400390625, -1.06640625, -0.5634765625, -0.306396484375, 0.40478515625, -0.546875, 0.217041015625, -1.0732421875, -0.77978515625, 1.0791015625, -0.1905517578125, 0.436279296875, 0.1763916015625, 0.7666015625, -1.125, -0.779296875, 0.359619140625, -1.4697265625, 1.111328125, -0.454345703125, 1.6513671875, 0.75244140625, -0.615234375, 0.546875, 1.138671875, -0.1722412109375, -0.9375, -0.288330078125, 0.095947265625, -0.85888671875, 0.016754150390625, 0.74072265625, 0.19580078125, -0.60546875, -0.463623046875, 0.0318603515625, -0.19775390625, 0.0296630859375, 0.443359375, 0.89599609375, 0.63232421875, -1.5009765625, -1.9609375, -0.059783935546875, 1.2880859375, -0.376708984375, 0.474609375, 3.248046875, -0.50830078125, -0.462646484375, -1.4248046875, -0.41796875, 1.181640625, 1.787109375, -0.377685546875, -0.047943115234375, 2.09375, -0.4833984375, -2.111328125, 0.720703125, 0.497314453125, 0.109375, 0.345458984375, 1.5498046875, -1.5537109375, -0.34765625, 0.326416015625, -1.2822265625, 0.343017578125, 0.473876953125, -0.032684326171875, -0.295654296875, -1.705078125, -0.36572265625, 0.9208984375, -0.9228515625, -1.83984375, 0.60595703125, -2.2109375, -0.65234375, -1.3876953125, -0.9345703125, -0.0855712890625, -0.82568359375, 1.4501953125, 1.2578125, -0.92724609375, -0.9208984375, 0.568359375, 1.2197265625, 1.4248046875, -0.84130859375, 1.4541015625, 0.250244140625, 0.40673828125, -0.177978515625, 0.1669921875, -1.5517578125, 1.4931640625, 0.42724609375, -0.26123046875, -0.361572265625, 0.80224609375, -0.419189453125, 1.515625, -2.140625, 0.471923828125, 0.05224609375, 0.79296875, 0.8154296875, -0.2744140625, 0.4951171875, 2.5703125, 0.81640625, -1.5908203125, 1.431640625, -0.89697265625, -0.1827392578125, -0.88232421875, -1.357421875, -1.0595703125, 0.09716796875, -1.271484375, -0.028411865234375, 1.28125, -1.0986328125, -0.80810546875, 2.228515625, -0.12744140625, 0.30908203125, -0.2342529296875, -1.298828125, 0.4658203125, 0.04901123046875, -1.1083984375, 0.4599609375, 1.2880859375, 1.080078125, -0.52392578125, 0.331787109375, 1.4990234375, 0.52197265625, -1.5634765625, 0.78759765625, 1.0810546875, 1.0986328125, 0.8115234375, 1.14453125, -1.5478515625, -1.095703125, 2.55859375, -1.08203125, 0.86328125, -0.167724609375, -0.947265625, -0.900390625, 0.802734375, -0.196044921875, -0.705078125, 0.00534820556640625, -1.373046875, -0.002147674560546875, -0.7294921875, -0.303955078125, 0.64794921875, -0.488037109375, 1.0302734375, -0.2091064453125, 0.7734375, -0.43359375, -0.74462890625, -1.8642578125, -0.283203125, -0.30126953125, 1.373046875, 0.90966796875, 0.2073974609375, -0.299072265625, -0.13818359375, -0.98486328125, 0.164794921875, 0.259033203125, -1.6484375, -1.224609375, -0.4755859375, -0.337646484375, -0.87060546875, -1.1640625, -1.0498046875, 0.0214080810546875, 1.9755859375, -0.0264739990234375, -1.40625, -0.9697265625, -0.79150390625, 0.5849609375, 0.26611328125, -0.126708984375, 0.51416015625, -1.912109375, 0.017974853515625, 1.22265625, 0.73193359375, -0.498291015625, -0.69970703125, -1.4111328125, 1.353515625, -2.3046875, 0.0687255859375, 1.1396484375, -0.828125, -0.27294921875, -1.228515625, -0.88525390625, 0.1651611328125, -0.8779296875, -0.54638671875, 0.033447265625, -0.39599609375, -0.80419921875, 1.310546875, -1.2333984375, -0.30224609375, -1.333984375, 0.273681640625, -1.232421875, -0.994140625, 0.034515380859375, -0.65283203125, -0.1185302734375, -0.61279296875, -0.5556640625, -0.46435546875, -0.60498046875, 0.29541015625, 0.59765625, -0.83544921875, -0.73779296875, -0.03216552734375, 0.203125, 0.33984375, -0.382080078125, 1.6005859375, 1.28125, -0.167236328125, 0.70166015625, -1.1435546875, 0.484375, 1.0439453125, -0.8701171875, 0.9658203125, 0.06976318359375, -1.4638671875, -0.69580078125, 0.236572265625, 0.8154296875, 0.013031005859375, 0.82763671875, 0.08392333984375, 1.515625, 2.28515625, 1.1826171875, 0.402099609375, 0.2421875, -0.189697265625, 1.20703125, -0.43994140625, -1.1767578125, 1.001953125, 2.365234375, -1.2685546875, -1.3154296875, 1.63671875, -1.41796875, -1.2919921875, 0.69921875, 1.314453125, 2.1640625, -0.409423828125, -0.1021728515625, -0.0277557373046875, -1.05078125, -0.541015625, -0.2236328125, 0.06304931640625, 1.4423828125, -0.0689697265625, 0.98681640625, -1.2568359375, 0.68359375, -0.58740234375, 0.178955078125, 0.61572265625, -0.974609375, -0.82470703125, -1.1572265625, -0.465087890625, -0.60009765625, -0.07305908203125, -0.10247802734375, 1.4306640625, -0.880859375, 0.457763671875, 1.171875, -0.53173828125, -0.4619140625, 0.38720703125, 0.5244140625, 0.2178955078125, -2.931640625, 0.38916015625, -1.2099609375, -0.292724609375, -0.50244140625, 0.9443359375, 0.63720703125, 1.0234375, 0.292236328125, 0.1298828125, 2.4453125, 0.875, -1.53515625, 0.1513671875, 0.2822265625, -0.01546478271484375, 0.392333984375, 0.455322265625, -1.0478515625, -0.303466796875, 0.5869140625, 1.134765625, 1.8115234375, 0.82373046875, -1.580078125, 0.36962890625, -1.802734375, -0.5498046875, -0.0310516357421875, -0.70947265625, 0.1654052734375, 1.8720703125, 0.1793212890625, -0.7314453125, -0.00919342041015625, -0.1741943359375, -0.385498046875, -0.3720703125, 0.8525390625, -1.6455078125, -1.7998046875, -1.9111328125, 1.427734375, -0.324951171875, 0.108642578125, 0.2049560546875, -0.52197265625, -1.6865234375, -0.6787109375, -0.1741943359375, 1.248046875, 0.462646484375, 0.396728515625, -0.9638671875, -1.640625, 0.78955078125, 0.202392578125, 0.35302734375, -1.060546875, -1.3701171875, 0.9287109375, -0.60302734375, 1.6611328125, -0.80810546875, -1.748046875, 2.025390625, 0.1278076171875, -0.399658203125, -1.1455078125, -0.4697265625, -0.90966796875, 0.344482421875, -0.94189453125, -2.283203125, -0.040374755859375, -1.0595703125, 0.15869140625, 0.2374267578125, 0.06268310546875, -0.5888671875, 1.466796875, 0.31787109375, -1.587890625, -0.07568359375, -0.1771240234375, -0.023956298828125, 0.81787109375, -0.60498046875, -1.390625, 2.296875, -1.2138671875, 0.748046875, -0.492919921875, -0.052520751953125, 1.07421875, -1.3544921875, 0.365234375, -0.58642578125, -0.465576171875, -0.1826171875, -0.43603515625, 1.5625, 1.90625, -0.796875, 1.48046875, 0.58984375, 0.253173828125, 1.3212890625, 0.55126953125, 0.88623046875, -1.27734375, 1.689453125, 0.5947265625, 0.0369873046875, -0.26123046875, -1.3193359375, 0.5712890625, -0.64990234375, -0.77734375, -0.35986328125, -0.435546875, -2.05078125, 2.330078125, -0.226806640625, -0.37744140625, 0.9462890625, 1.09765625, -0.5712890625, 0.7705078125, 0.111083984375, -0.35400390625, -0.69287109375, 0.82421875, -3.0625, 1.0341796875, -0.0255889892578125, -0.787109375, -0.6513671875, 1.4853515625, 2.19921875, -0.96142578125, -0.578125, -0.75634765625, 1.109375, 0.576171875, 0.0357666015625, 0.72216796875, -0.198974609375, -2.171875, 1.01953125, 0.908203125, 0.1156005859375, -0.2484130859375, -1.359375, -1.05859375, 1.9951171875, -1.203125, -0.01363372802734375, 0.560546875, 0.2218017578125, 0.327392578125, 2.626953125, 0.458740234375, 2.8984375, 0.459716796875, 0.43115234375, -0.67724609375, 0.99169921875, -1.2861328125, 0.55517578125, 0.219482421875, -2.037109375, -0.61572265625, -0.237548828125, 0.42041015625, -0.00968170166015625, -0.1297607421875, 0.927734375, 0.44091796875, 0.65478515625, -0.869140625, 0.53173828125, -0.73828125, -0.7783203125, -0.356689453125, -1.5009765625, -0.427978515625, 0.26513671875, 0.97021484375, -0.323974609375, 0.238037109375, -0.10528564453125, 1.3203125, -0.98193359375, 2.9140625, -0.970703125, 2.125, 0.397705078125, -1.2451171875, 0.124755859375, -0.572265625, 0.27099609375, -1.1240234375, -0.970703125, -1.087890625, 0.90478515625, -1.517578125, 0.408447265625, 0.044769287109375, 1.8076171875, 0.7666015625, -0.040435791015625, 1.3115234375, 0.045257568359375, -0.96240234375, 0.828125, 0.556640625, 1.306640625, -0.5771484375, 0.6376953125, -1.287109375, 1.0859375, -0.32958984375, 1.0927734375, 1.09765625, -1.158203125, 0.533203125, -1.1318359375, -1.599609375, -0.31884765625, 0.439453125, -0.89306640625, -0.03265380859375, 1.0791015625, -1.1171875, -0.322265625, -0.380859375, 1.55078125, 0.47900390625, 2.2109375, -1.9453125, 0.8232421875, -1.455078125, 1.017578125, -0.478759765625, -0.7041015625, 1.6552734375, -0.1744384765625, -0.76806640625, -0.4326171875, 1.1328125, 1.4765625, 0.0633544921875, -1.197265625, -0.423828125, 0.416015625, -0.8271484375, -1.9296875, -0.1549072265625, 0.61083984375, 1.1279296875, 0.568359375, -0.05987548828125, 0.282470703125, -1.640625, 0.8974609375, 0.50634765625, -0.27978515625, -0.1844482421875, -0.8271484375, 0.81884765625, -2.23046875, 0.1484375, -0.1295166015625, 0.70947265625, 1.015625, -1.25390625, -0.9189453125, 0.56689453125, -0.171630859375, -0.20263671875, 0.69677734375, 1.12890625, 1.4501953125, -1.451171875, -2.904296875, 1.5029296875, 1.125, -1.9873046875, -0.84375, 1.974609375, -0.02081298828125, 1.2099609375, -1.2529296875, -0.79638671875, -0.623046875, -0.7861328125, -1.5146484375, -0.56298828125, -0.5498046875, 0.497314453125, 2.29296875, -0.0106658935546875, -0.66943359375, 1.5654296875, -0.79248046875, 0.4267578125, -1.7744140625, -2.166015625, -0.9091796875, -1.474609375, 0.7900390625, 0.403076171875, 1.892578125, -0.0693359375, -0.0271453857421875, -0.4072265625, 1.455078125, 0.423583984375, 0.44189453125, -1.228515625, -1.0673828125, -0.307861328125, 2.712890625, -0.496337890625, 1.2431640625, -0.79052734375, 2.19921875, -0.66064453125, 0.77099609375, -0.499267578125, 0.212158203125, 0.0970458984375, 0.912109375, 0.157958984375, -0.54248046875, 0.2064208984375, 0.0041351318359375, 0.62939453125, -0.278076171875, 0.7080078125, -0.97265625, 0.41650390625, -0.927734375, -0.48681640625, 0.144775390625, -1.0166015625, 1.1796875, 0.591796875, -2.177734375, 1.4345703125, -0.2401123046875, 1.9990234375, -1.33203125, -0.48046875, 1.4619140625, -0.29052734375, 1.0595703125, -0.681640625, 1.44921875, 1.06640625, 0.67529296875, 0.481689453125, 1.697265625, 0.64990234375, 0.0223236083984375, 1.3037109375, 0.495849609375, -0.72265625, -1.19140625, -0.5439453125, 0.10260009765625, -1.0400390625, 0.9404296875, -0.6982421875, -0.142578125, 0.650390625, -0.525390625, -0.43896484375, 0.27294921875, -0.211669921875, 0.6689453125, -1.01953125, 0.64111328125, 0.943359375, 0.1978759765625, 0.9248046875, -2.060546875, 0.4580078125, -1.126953125, -1.24609375, 0.1409912109375, 1.64453125, -2.390625, -0.283447265625, -1.3017578125, 0.1695556640625, 0.1470947265625, -0.5322265625, -0.2474365234375, 1.2353515625, -0.77685546875, -1.28515625, 0.6630859375, 1.5751953125, -0.8486328125, -0.449951171875, -0.313232421875, 0.478759765625, -0.08941650390625, -1.8984375, -2.013671875, -0.2156982421875, 1.4072265625, -1.1494140625, 1.2490234375, 0.482666015625, -0.357177734375, -0.9609375, 0.84765625, -1.1455078125, 0.3046875, -0.471923828125, -1.6357421875, 0.405029296875, -1.548828125, 0.55712890625, -1.046875, -0.66845703125, -1.6748046875, -0.44482421875, 1.2607421875, -0.91015625, -0.51513671875, -1.2626953125, -1.404296875, 0.330810546875, 0.199462890625, 1.8359375, 1.296875, 1.0166015625, 1.1171875, -0.42529296875, 0.403564453125, 1.58203125, -1.578125, 0.45947265625, -0.98193359375, -0.1669921875, 0.0178070068359375, 1.58984375, -0.53662109375, 3.74609375, 0.055908203125, 0.796875, 1.0546875, 0.1708984375, 0.76171875, 0.2166748046875, 1.19140625, -1.4189453125, -0.391845703125, 0.34521484375, 0.96240234375, 0.78125, 1.3779296875, -0.5146484375, 2.162109375, 1.521484375, -1.39453125, 3.193359375, -1.369140625, 0.515625, 0.818359375, 0.25048828125, -0.10357666015625, 0.226318359375, 0.60546875, -0.8037109375, 1.2685546875, 1.5, -1.5966796875, -0.55908203125, -0.1082763671875, -2.32421875, -0.57861328125, 0.5654296875, 0.416015625, -0.4599609375, -0.6708984375, -0.10321044921875, -0.302734375, 0.05804443359375, 0.214599609375, -1.1435546875, 0.70947265625, 0.58984375, 0.8798828125, -0.287109375, 0.42431640625, -1.783203125, 0.425537109375, -1.306640625, 1.0791015625, 1.1171875, -1.0830078125, 1.01953125, -0.30029296875, -0.8994140625, -0.30322265625, 0.499755859375, -1.1962890625, -0.4443359375, -0.304931640625, -2.048828125, 0.43017578125, 1.5595703125, -0.34912109375, -1.095703125, -0.1898193359375, -1.30078125, -0.2479248046875, -0.79931640625, -0.1978759765625, -0.73193359375, -1.14453125, -0.1302490234375, -0.253662109375, -0.12347412109375, -0.50048828125, 0.43505859375, 0.13720703125, -0.2373046875, 1.275390625, 0.1915283203125, 2.111328125, 0.77099609375, 0.1851806640625, -0.2406005859375, 0.035919189453125, 2.046875, -0.74169921875, -0.01141357421875, 2.5078125, 0.28466796875, 0.564453125, -0.9296875, -0.31884765625, -0.53173828125, -0.96728515625, -0.3212890625, -0.1732177734375, 0.447509765625, 0.67236328125, 1.138671875, 0.027801513671875, -1.142578125, 1.3037109375, -0.0088958740234375, -1.642578125, -0.80322265625, 0.326904296875, 0.9873046875, -0.59423828125, 0.177490234375, 0.09063720703125, -2.015625, 1.4912109375, 0.1759033203125, -1.1943359375, 2.001953125, -0.99267578125, 1.123046875, 1.6533203125, 1.2822265625, 1.3359375, 0.042083740234375, -0.89697265625, -1.662109375, 0.7265625, 1.611328125, 1.8525390625, 1.208984375, -1.21875, 0.142578125, -0.5517578125, -1.720703125, -0.077392578125, 1.6787109375, -0.75439453125, 0.1907958984375, -0.75146484375, -0.22021484375, -0.611328125, 0.068603515625, 1.69921875, -1.4375, -0.826171875, -0.8232421875, -1.099609375, -1.029296875, 0.99072265625, 0.332275390625, -1.3671875, -0.8935546875, -0.364990234375, -1.501953125, 1.265625, -1.208984375, -0.546875, -0.5009765625, -0.9033203125, 0.25390625, 1.83203125, -1.626953125, -0.16015625, 0.72119140625, -0.6015625, -0.254638671875, -0.1932373046875, -1.2578125, 2.064453125, -0.017303466796875, -0.465087890625, 0.0034770965576171875, -0.41796875, -0.99755859375, 0.401123046875, 0.480712890625, -0.78857421875, 0.12164306640625, -0.53662109375, 0.284423828125, 0.876953125, 0.10589599609375, -0.34375, -0.63232421875, 1.623046875, -0.034332275390625, -0.1585693359375, 0.67724609375, -1.6279296875, 1.435546875, 1.0830078125, -1.201171875, 1.6455078125, -1.5439453125, -0.2939453125, 0.424560546875, -0.401123046875, -0.353759765625, 0.3154296875, 1.5048828125, -1.26953125, 0.65966796875, -1.0126953125, 0.2015380859375, 0.84716796875, 1.8095703125, 0.796875, -0.21923828125, 0.751953125, 2.310546875, -0.137451171875, -0.057098388671875, 1.7998046875, 0.0968017578125, -1.0830078125, 1.072265625, 0.7890625, -1.6748046875, -0.63427734375, -0.96435546875, 1.2080078125, -0.88330078125, 0.1072998046875, 1.3369140625, 0.07867431640625, 0.54541015625, 0.87890625, 1.880859375, -1.005859375, 1.5654296875, -1.2255859375, -0.159912109375, 1.1240234375, 0.1732177734375, 0.66748046875, -0.4189453125, -0.12841796875, 0.433349609375, -0.8701171875, 0.34814453125, -0.0011425018310546875, 1.6328125, 1.3505859375, 0.77099609375, 0.2890625, -0.1224365234375, -1.744140625, 0.52587890625, -2.20703125, -0.61181640625, 1.2021484375, 0.6640625, 1.943359375, 0.6982421875, 1.76953125, 0.7666015625, -1.7568359375, -0.06494140625, -1.2197265625, 0.607421875, 0.78857421875, 0.11090087890625, -0.91064453125, -0.8984375, -1.0859375, 0.83837890625, -0.623046875, -0.491943359375, 0.68701171875, -1.1171875, -0.303955078125, 1.53515625, 0.441650390625, -0.7275390625, -0.377197265625, 0.6025390625, 0.513671875, 0.3740234375, -0.4189453125, -0.65966796875, -1.5908203125, 0.2822265625, -0.25146484375, 0.426025390625, -0.6337890625, -2.216796875, -1.2431640625, -0.257080078125, 1.2451171875, 0.73828125, 0.99462890625, 0.96533203125, -0.2900390625, -1.8427734375, -0.1258544921875, 0.7275390625, -0.07525634765625, -0.58154296875, -0.912109375, -1.1650390625, -0.97705078125, 1.1240234375, 0.0711669921875, -0.435302734375, 1.10546875, 0.62158203125, 0.5068359375, -0.56640625, -0.3681640625, 0.026611328125, 0.12493896484375, 0.6474609375, -0.748046875, -1.1103515625, 0.271240234375, -0.79833984375, 0.1370849609375, 0.305419921875, 2.119140625, 0.86572265625, 0.432373046875, -1.138671875, 0.04827880859375, 0.794921875, -0.09906005859375, -0.109619140625, 0.406005859375, -0.1265869140625, -0.5400390625, 0.06585693359375, -0.99560546875, 0.97509765625, -1.09375, -0.8896484375, -0.650390625, -0.9326171875, -0.9287109375, -0.12469482421875, -1.8896484375, 0.296630859375, 0.326416015625, -0.61572265625, -0.03863525390625, -0.138916015625, -1.31640625, 0.4345703125, 1.7919921875, 0.74755859375, -0.81689453125, 0.34521484375, 1.67578125, -0.79150390625, 0.419921875, 1.11328125, 0.43017578125, -1.548828125, -0.2322998046875, -0.397216796875, -1.8271484375, 1.3486328125, 0.2666015625, -2.296875, 1.11328125, -0.472412109375, -0.0212249755859375, -0.0175628662109375, 0.268310546875, -0.552734375, 1.1875, -0.67138671875, -1.0556640625, 1.4423828125, -0.90625, 1.76953125, 0.61181640625, -0.0948486328125, -0.1566162109375, -0.875, -0.7197265625, -0.2266845703125, -1.0732421875, -1.232421875, 0.1270751953125, -0.26220703125, -0.97509765625, 0.990234375, -0.06463623046875, 0.61279296875, -0.451904296875, -0.2000732421875, -0.9716796875, 0.372802734375, 2.798828125, 0.292236328125, 0.91015625, -0.133544921875, 1.919921875, 1.4296875, 1.1162109375, 2.001953125, 0.18115234375, -0.46533203125, -0.7890625, 0.7177734375, 0.1771240234375, -1.787109375, -1.4248046875, -0.038360595703125, 0.90576171875, 1.9619140625, 2.5234375, -1.2978515625, 0.72802734375, 0.04376220703125, -0.84326171875, 0.31884765625, 0.93310546875, 0.1566162109375, -0.35400390625, -0.72265625, -1.4765625, -0.1649169921875, -0.1988525390625, -0.04156494140625, -1.2001953125, -0.810546875, -1.6962890625, -0.802734375, 0.90966796875, 1.5947265625, -0.50146484375, -1.3330078125, 0.556640625, -0.400634765625, -1.41796875, -1.3330078125, -0.38818359375, -1.1298828125, -1.8994140625, -0.99853515625, -0.96630859375, -0.51220703125, 1.1943359375, -0.08380126953125, -1.1474609375, -1.548828125, -0.408203125, 0.79638671875, 0.52685546875, 1.8701171875, 0.51806640625, 0.27197265625, -0.64013671875, -0.262451171875, -0.57861328125, 1.3466796875, 0.8310546875, -0.1783447265625, -0.27392578125, -1.31640625, -1.40625, -1.0927734375, 0.410400390625, -0.166259765625, 0.58203125, -1.4775390625, -0.456298828125, 0.53955078125, -0.921875, 0.7626953125, -1.24609375, 0.56640625, 0.68603515625, -0.57275390625, 0.423828125, -0.2156982421875, 0.52783203125, 1.501953125, -2.1015625, 1.1435546875, 0.346435546875, 1.13671875, -2.134765625, 1.0595703125, -0.467529296875, 1.634765625, -1.2373046875, 0.91064453125, -0.765625, 0.19384765625, 0.301513671875, -0.28125, 0.68994140625, 0.75390625, 1.6875, -1.9228515625, -0.11285400390625, -0.42236328125, 0.849609375, 0.81591796875, -0.28662109375, 0.221923828125, -0.046478271484375, 0.7177734375, -0.489501953125, -1.330078125, 0.77197265625, -0.3037109375, -0.7353515625, 1.154296875, -1.8330078125, -2.1796875, -0.392578125, -1.330078125, -0.060333251953125, -0.0640869140625, 0.0258331298828125, 1.4990234375, 0.2391357421875, -1.009765625, 0.0675048828125, -0.8759765625, -0.78955078125, -0.64599609375, 1.87109375, 1.9365234375, -0.1597900390625, 0.92431640625, -0.494873046875, 1.70703125, -0.1982421875, 2.025390625, -2.34765625, -1.2421875, -0.456787109375, 0.06280517578125, 0.27880859375, -0.2724609375, 2.515625, 0.033355712890625, -0.3359375, -0.43408203125, 0.54931640625, -0.007640838623046875, -0.0090179443359375, -0.435546875, -1.169921875, 1.041015625, -0.66064453125, 1.0322265625, 0.5068359375, -0.52099609375, 0.5400390625, -2.13671875, 0.3466796875, 0.069580078125, -1.240234375, 0.0802001953125, -1.0830078125, 0.32373046875, 0.408447265625, 1.83984375, 0.806640625, 0.90673828125, 0.64501953125, 1.0244140625, -1.3115234375, -0.81787109375, -0.498779296875, -0.193359375, -1.9892578125, 0.5888671875, 0.16015625, -0.6083984375, 1.7099609375, 0.336181640625, -2.046875, -0.53515625, -1.259765625, 1.4296875, -0.048492431640625, 0.39306640625, -0.347900390625, -1.291015625, -2.740234375, 1.580078125, -1.2451171875, -0.62353515625, 0.435302734375, 0.88720703125, 1.65625, 2.0078125, -0.52001953125, 0.8115234375, -1.9873046875, -1.5400390625, -0.468017578125, -0.43115234375, -1.3046875, -0.375, -0.97509765625, 0.114013671875, 0.245849609375, -2.197265625, 0.09368896484375, 0.86279296875, -0.220458984375, 0.58056640625, -0.51220703125, 0.9443359375, -0.403076171875, 0.29736328125, 0.76416015625, -0.95361328125, -0.212890625, 0.5859375, -0.2169189453125, -0.87060546875, -0.359375, -1.6103515625, 0.032684326171875, 0.77392578125, -1.375, 0.9501953125, -0.85888671875, 1.326171875, 0.2088623046875, 0.6279296875, 0.059967041015625, -1.75390625, -1.095703125, 1.626953125, 0.0233154296875, 0.80126953125, -1.1826171875, 0.10986328125, -1.3486328125, -1.0048828125, -0.409912109375, -0.6416015625, -0.6123046875, 1.3583984375, 1.6396484375, 0.378173828125, 0.2222900390625, 1.37109375, -2.025390625, -0.1263427734375, -1.01171875, 0.9287109375, -2.05859375, -0.92236328125, -0.355712890625, 0.198974609375, -0.169189453125, -0.92578125, -0.149658203125, 0.83740234375, -0.71435546875, 1.8974609375, 0.4775390625, 0.21044921875, 1.5478515625, 0.04107666015625, 2.5546875, -0.5517578125, -0.2335205078125, -0.05865478515625, -0.2255859375, 0.81591796875, 1.9326171875, 0.2208251953125, -1.1201171875, 0.0027618408203125, 0.95556640625, -0.79736328125, -0.62109375, 0.73388671875, -0.57177734375, -0.493408203125, -2.099609375, -0.8564453125, 0.0877685546875, -1.2900390625, -1.1767578125, 1.7041015625, -0.288330078125, 0.99365234375, -0.59619140625, -0.06451416015625, 0.34716796875, -0.76904296875, 0.5087890625, 0.04718017578125, -0.8291015625, -0.72900390625, -0.671875, -0.1934814453125, 1.6435546875, 0.81640625, -0.609375, -0.028045654296875, 0.33984375, -0.59716796875, -0.81787109375, 3.005859375, -0.65576171875, 0.26611328125, 0.87646484375, -0.374755859375, -0.1614990234375, -1.60546875, 0.279541015625, 0.49658203125, 0.29638671875, 0.92333984375, -0.7958984375, -0.6103515625, 2.71484375, -0.0009760856628417969, -0.14404296875, 1.3857421875, 0.8447265625, -1.0830078125, 0.0145416259765625, -0.244384765625, 0.7001953125, -0.293212890625, -0.78466796875, 0.07281494140625, -1.2783203125, -1.3681640625, 0.10888671875, -0.17822265625, 2.62109375, 0.033599853515625, -0.400634765625, -0.755859375, -0.51123046875, 0.63623046875, 1.0166015625, 0.310791015625, -0.00926971435546875, 1.44140625, 1.0576171875, 0.70556640625, 0.1788330078125, -0.482666015625, -2.25390625, -0.7626953125, 1.3544921875, 0.65625, 0.79736328125, -0.3671875, 0.42333984375, 0.038970947265625, 0.03265380859375, -0.64453125, 1.5361328125, -1.3115234375, -0.205078125, -0.5888671875, -0.12005615234375, -1.0322265625, 0.24658203125, -0.8896484375, 0.9951171875, 0.37841796875, 0.650390625, -0.1475830078125, -1.17578125, -0.654296875, -0.044219970703125, -0.442626953125, 1.0341796875, -1.130859375, -0.72119140625, -0.39306640625, 0.73779296875, -0.255126953125, 2.119140625, -1.21484375, -1.78515625, -0.94091796875, -1.263671875, -1.353515625, -0.96533203125, 0.4326171875, -0.92919921875, -1.943359375, 0.36279296875, -1.060546875, -1.1396484375, -1.064453125, 0.06201171875, 0.298828125, 1.8408203125, 0.07623291015625, -0.13623046875, -1.998046875, -0.060882568359375, -1.7333984375, -0.58544921875, 0.44873046875, -0.46826171875, 0.244140625, -0.83203125, -1.2900390625, 1.3388671875, 0.415283203125, -1.0107421875, -0.267822265625, 1.3359375, -0.62939453125, 1.3515625, -0.953125, 1.7705078125, 0.70166015625, 0.07861328125, -1.951171875, -0.3212890625, 0.94921875, -0.39697265625, -0.6513671875, 2.087890625, 0.58740234375, -1.7939453125, -0.6982421875, 1.2041015625, -0.72900390625, -2.37890625, -0.80126953125, 1.8623046875, -1.884765625, -0.2017822265625, -2.306640625, -2.099609375, -0.438720703125, 2.025390625, -0.892578125, -0.61376953125, 0.81884765625, -0.52978515625, 0.9521484375, -0.79541015625, -0.7958984375, 1.4033203125, 0.4541015625, -0.69921875, -1.634765625, -0.5185546875, -0.26904296875, 0.54443359375, 0.7158203125, -1.330078125, 1.5048828125, 0.72900390625, 2.732421875, -0.437255859375, 0.366455078125, -1.341796875, -0.6240234375, -0.43603515625, -0.1834716796875, 0.12469482421875, -1.6923828125, -1.6201171875, -2.447265625, -0.4931640625, 1.287109375, -1.3701171875, -1.3564453125, 2.458984375, 1.0126953125, -0.418212890625, -0.81103515625, -1.1240234375, -1.396484375, 0.7060546875, 0.152099609375, -0.31982421875, 0.5263671875, 0.389892578125, 0.265625, -0.93505859375, -0.5947265625, 0.93310546875, -1.0888671875, -0.1295166015625, 0.47216796875, -1.751953125, 1.0859375, -1.0478515625, -1.169921875, -0.058807373046875, 0.59619140625, -3.044921875, -0.85888671875, 0.06207275390625, 0.87060546875, -0.541015625, -0.99560546875, 1.037109375, 0.92333984375, -0.603515625, -0.30078125, 0.11993408203125, 0.136474609375, 0.86669921875, 0.50439453125, 0.7041015625, -0.65380859375, 1.3642578125, -0.33251953125, -0.1685791015625, -0.6328125, 0.1112060546875, 1.525390625, -0.0174713134765625, -0.0311126708984375, -0.0665283203125, 0.17626953125, -1.896484375, -1.130859375, 0.166259765625, -0.58056640625, 1.685546875, -1.5048828125, 0.611328125, -0.30322265625, -0.1607666015625, 0.83544921875, -0.253662109375, -0.46533203125, -0.399658203125, -0.486328125, -1.4189453125, 1.3251953125, 0.8935546875, -0.1905517578125, -0.463623046875, 1.3466796875, 1.072265625, 2.333984375, -1.505859375, 0.085205078125, 2.212890625, 0.7822265625, 1.6142578125, -0.81787109375, -0.765625, 0.1961669921875, 1.53515625, -0.2281494140625, 1.3056640625, -0.73291015625, -0.123779296875, -0.1866455078125, 0.369384765625, 0.50390625, 0.127685546875, 0.55224609375, 0.52099609375, -0.11541748046875, -0.420166015625, -0.052215576171875, 1.1396484375, 1.892578125, 1.2607421875, -0.83447265625, 1.765625, -0.56494140625, -0.350830078125, 0.2744140625, -1.060546875, 0.38623046875, 2.1484375, 0.51025390625, 2.6953125, 0.71728515625, -0.4150390625, 0.4013671875, -1.2353515625, 0.16943359375, 0.175537109375, -0.70263671875, -0.352294921875, -1.5029296875, 0.1571044921875, 0.7919921875, -1.3173828125, -0.60595703125, -0.50732421875, -0.08941650390625, 0.0780029296875, -0.9755859375, -0.0440673828125, 0.80859375, 0.4248046875, 0.70849609375, -1.3974609375, 0.09735107421875, 0.68408203125, 0.79248046875, -0.1094970703125, -1.6943359375, -0.00478363037109375, 2.3984375, -0.544921875, -0.1016845703125, -0.224853515625, 0.1951904296875, -0.360107421875, -0.4150390625, 0.73583984375, 0.6328125, 0.6533203125, 1.130859375, -0.0731201171875, 0.39892578125, 0.004116058349609375, -0.58056640625, 2.375, -0.289794921875, -0.947265625, 0.01198577880859375, -0.95263671875, -1.9091796875, 0.578125, 0.371826171875, -0.8837890625, -1.5029296875, 0.051727294921875, 1.0859375, -0.1455078125, 0.224853515625, 1.37890625, -0.361328125, -0.322265625, -0.60546875, 1.8095703125, 0.083984375, -0.052001953125, -0.72900390625, -1.1337890625, -0.64208984375, -1.2802734375, 0.4853515625, 0.421630859375, 0.978515625, 0.08087158203125, 0.2340087890625, 0.5771484375, -1.6435546875, -1.865234375, 0.332763671875, 1.984375, -1.1484375, -0.28125, 0.56298828125, 0.75537109375, -0.8916015625, -1.447265625, 0.44970703125, 0.30078125, 2.228515625, 0.282958984375, 1.2021484375, -1.25, -1.3564453125, 2.279296875, -0.12139892578125, 1.4453125, 0.810546875, -0.5341796875, 0.814453125, -0.329345703125, -0.9228515625, -0.89794921875, 0.6630859375, -0.298095703125, -1.09765625, -0.8291015625, -0.448486328125, -0.40185546875, 1.361328125, 0.95556640625, -0.350830078125, 3.525390625, -0.6845703125, 1.16015625, -0.7392578125, 2.00390625, -0.1585693359375, 0.29345703125, -0.56884765625, 0.99658203125, 1.6650390625, 0.935546875, 0.41943359375, 0.1038818359375, -1.4404296875, -0.6279296875, -0.2476806640625, -0.42333984375, -1.322265625, -1.58984375, 2.0234375, -0.87353515625, 0.28466796875, 0.60791015625, 2.29296875, -0.49853515625, 1.119140625, 0.9150390625, -0.55029296875, 0.366943359375, -0.58154296875, -0.95361328125, 0.22021484375, 0.94580078125, 1.6025390625, -0.369384765625, -0.59912109375, -0.31689453125, 0.1715087890625, -0.62060546875, 0.326171875, -0.1591796875, 0.12457275390625, 1.015625, -1.203125, -0.425537109375, 0.3388671875, -0.5673828125, -0.2386474609375, 0.6923828125, -1.4638671875, 1.5458984375, -0.84716796875, -0.295654296875, 0.1859130859375, 0.84326171875, 1.8037109375, -1.298828125, -1.72265625, -0.1591796875, -0.2880859375, -1.5703125, -0.86865234375, 0.931640625, -1.3134765625, -0.50634765625, -0.2239990234375, -0.046844482421875, 0.261474609375, -0.355224609375, -0.2008056640625, 0.62158203125, -0.61376953125, 1.48046875, -0.492431640625, 0.2413330078125, -0.70849609375, -0.437744140625, 2.099609375, -1.158203125, -2.3515625, -1.77734375, -0.607421875, -0.1561279296875, -0.2071533203125, 1.083984375, -0.64990234375, 0.1275634765625, -0.007488250732421875, -2.068359375, -1.71875, -1.5390625, 2.56640625, -0.41357421875, -0.0255126953125, -0.5087890625, 0.78125, 0.0540771484375, 0.74755859375, -1.6875, -0.501953125, -0.61376953125, -0.89501953125, 1.6025390625, 1.544921875, 0.40185546875, -2.5, 1.00390625, 1.71875, -0.54541015625, -0.14892578125, 0.78076171875, 0.06982421875, -1.0625, 0.300537109375, -1.134765625, 1.1513671875, 0.08349609375, -1.00390625, -0.255126953125, 2.017578125, 0.233642578125, 0.67236328125, 1.1982421875, 0.57373046875, 0.09814453125, -0.1763916015625, 0.9267578125, -0.791015625, -0.1737060546875, 1.2109375, -0.323974609375, -0.203857421875, 0.6943359375, -0.73388671875, -1.869140625, 2.076171875, 0.8349609375, 1.30859375, -0.08050537109375, 0.34326171875, 0.78076171875, 0.39501953125, -1.2119140625, -0.449462890625, -0.7265625, 0.0020427703857421875, 0.027008056640625, -0.74609375, -2.494140625, -2.201171875, 0.61474609375, -1.919921875, 0.2880859375, -0.462646484375, 0.448486328125, 0.325927734375, -1.5517578125, 0.78515625, -0.25732421875, 0.0384521484375, -0.321533203125, -0.400634765625, -0.271484375, 0.60302734375, 0.405029296875, -0.2352294921875, 0.00495147705078125, -0.0506591796875, -0.60693359375, 2.4140625, -0.2325439453125, -0.51025390625, -0.888671875, -1.18359375, 1.0966796875, -1.42578125, 1.1728515625, 1.1015625, -1.4013671875, 1.1435546875, -0.11883544921875, 0.56298828125, -1.6953125, -0.98291015625, -0.1922607421875, 1.611328125, -1.267578125, -1.2548828125, -0.09600830078125, 1.58984375, -0.79736328125, -0.2744140625, -0.34228515625, 0.28369140625, 0.0208740234375, 1.724609375, 2.001953125, -2.013671875, 0.8125, -0.0738525390625, -0.84521484375, -0.08233642578125, -0.71484375, 0.90673828125, -0.444580078125, -1.9287109375, 0.40625, -0.53564453125, 1.126953125, 1.693359375, -1.0380859375, -0.6572265625, -1.056640625, -1.4609375, 0.62841796875, -0.80078125, -0.03692626953125, -0.6630859375, -1.2314453125, 0.20703125, 0.869140625, -2.337890625, 0.41845703125, -0.0938720703125, 0.404541015625, 0.224609375, -0.033477783203125, 1.08203125, 0.368896484375, -0.0184173583984375, -1.6123046875, -0.0904541015625, 1.38671875, -0.6015625, -0.9267578125, -0.693359375, 0.73095703125, -0.374755859375, 0.346435546875, 0.059295654296875, -0.489013671875, 1.0439453125, -0.9140625, 0.35107421875, -0.0501708984375, 0.19384765625, -0.245849609375, -0.53271484375, 0.277587890625, -1.2529296875, 0.229736328125, -0.11627197265625, 0.751953125, -0.0728759765625, -0.1864013671875, 0.9892578125, -1.0400390625, -0.58154296875, -0.412353515625, 1.0078125, -1.1455078125, -1.0859375, 0.31689453125, -0.2005615234375, -1.75390625, 0.391357421875, -0.447998046875, -0.2027587890625, 0.90283203125, -0.16943359375, 1.8935546875, -0.09051513671875, -0.09002685546875, 0.8232421875, 0.5185546875, 0.294921875, 1.1005859375, 1.431640625, 0.20849609375, -0.78466796875, 0.56298828125, -2.21484375, -0.045501708984375, -0.493408203125, -0.371826171875, 1.3720703125, -1.65625, 1.673828125, -0.1199951171875, 2.2890625, -0.177490234375, -1.6337890625, -0.80810546875, 0.4873046875, 0.5517578125, 1.7939453125, 1.1005859375, -0.5341796875, -0.9580078125, -0.689453125, 0.1885986328125, -0.611328125, -1.1318359375, 0.423828125, 1.462890625, -1.3388671875, 0.6123046875, 0.6787109375, 0.89599609375, -0.11065673828125, 0.81640625, -0.250244140625, 0.6962890625, 1.5419921875, -1.3779296875, 1.0009765625, 0.67724609375, -0.410888671875, 0.7802734375, -0.54296875, 0.468505859375, -1.0673828125, 1.0322265625, -1.2607421875, -0.33544921875, 1.396484375, -1.1259765625, 0.2763671875, -0.66064453125, -1.466796875, 0.63134765625, -0.9638671875, -0.466552734375, 0.07574462890625, 0.76220703125, -0.81005859375, 1.0185546875, 0.296875, -1.1884765625, -0.55078125, 0.004856109619140625, 0.9072265625, -0.63330078125, 1.083984375, 0.75732421875, -1.931640625, 1.9091796875, 0.1104736328125, 0.0219879150390625, 0.353515625, -0.0024013519287109375, 0.8798828125, -0.5341796875, -0.474853515625, 0.1318359375, 1.5634765625, -0.63134765625, -1.388671875, 0.0073394775390625, -1.5166015625, -0.1549072265625, 1.2490234375, 0.1937255859375, 0.70361328125, -0.019378662109375, 0.204345703125, 0.08709716796875, 0.347412109375, 0.478759765625, -0.2052001953125, -1.45703125, 1.52734375, -1.166015625, -0.290771484375, -0.66162109375, -0.9033203125, 0.41845703125, 0.1317138671875, -0.63037109375, 0.8349609375, -0.8046875, -2.0625, -0.58740234375, -1.6875, -1.447265625, 0.62255859375, -0.50927734375, 1.478515625, 0.5869140625, 0.1976318359375, -0.322265625, -1.1171875, -0.51904296875, -0.75830078125, -0.06121826171875, 1.4912109375, 1.2646484375, -2.51953125, 1.5859375, 0.1837158203125, -0.36474609375, 0.284423828125, 0.2044677734375, 1.203125, 0.7119140625, -0.71142578125, -1.2119140625, 0.67236328125, 0.74755859375, -0.708984375, -0.5390625, 0.7294921875, 0.51416015625, 0.6982421875, -1.6572265625, 0.60400390625, 1.0771484375, 0.7626953125, 0.043975830078125, -1.2060546875, 1.11328125, 1.705078125, -1.341796875, 0.5, -0.9013671875, 0.2005615234375, -0.8115234375, -0.29833984375, 1.1298828125, -1.0078125, -0.501953125, 0.2314453125, 0.00939178466796875, 0.1837158203125, -0.1663818359375, -1.6416015625, 1.9990234375, -2.306640625, 1.611328125, 0.9375, -1.23046875, 1.326171875, -0.7763671875, -0.4580078125, 1.6767578125, 0.073974609375, 0.08782958984375, -0.415771484375, -0.2919921875, 0.60400390625, 0.48095703125, 0.395751953125, 0.413330078125, -1.3388671875, 0.94775390625, 1.5576171875, 0.5302734375, 1.82421875, 0.78857421875, 1.01953125, -0.24560546875, -0.333984375, -0.95263671875, 1.080078125, 1.2333984375, -0.08209228515625, -0.2861328125, 1.1630859375, -0.943359375, 0.52392578125, -0.46533203125, -3.3359375, -1.330078125, 0.6748046875, 0.251953125, 1.783203125, -0.9326171875, -0.4677734375, 0.67724609375, -0.041473388671875, 1.2177734375, 0.38720703125, -1.6298828125, -0.578125, -0.191162109375, -0.771484375, -0.09991455078125, -1.080078125, 0.951171875, -0.66162109375, -0.199462890625, 1.515625, 0.5859375, 0.78173828125, -0.5322265625, -0.61328125, -1.037109375, -0.908203125, -0.040008544921875, 0.0384521484375, -1.47265625, -1.240234375, 1.287109375, 0.64697265625, 0.92236328125, -0.939453125, 1.0625, 0.9501953125, 1.0947265625, 1.8701171875, 0.1959228515625, 1.1318359375, -1.3369140625, -0.68408203125, -0.047576904296875, -0.11285400390625, 0.68603515625, 1.658203125, -0.1427001953125, 0.319091796875, 0.1414794921875, -0.22802734375, -2.435546875, -0.275390625, -1.783203125, -1.421875, -0.748046875, -1.5234375, -2.412109375, -1.4970703125, 1.30859375, 0.57763671875, -0.058746337890625, 1.83984375, -2.10546875, -1.076171875, -1.0234375, -0.2110595703125, 0.8779296875, 2.0078125, 2.037109375, -1.4150390625, -0.611328125, -0.327880859375, 0.169921875, 1.0361328125, -1.111328125, 1.8828125, -0.517578125, 0.57763671875, 0.84375, 1.923828125, -1.48828125, -0.445556640625, -0.97900390625, 1.162109375, -0.8779296875, 0.28369140625, 1.162109375, -0.0082550048828125, -0.275390625, -1.1533203125, 2.328125, -0.39697265625, 0.0770263671875, -0.21142578125, -1.3291015625, -0.80810546875, 1.1513671875, -1.423828125, 0.54150390625, 0.01123046875, -1.5166015625, 0.32666015625, 0.011932373046875, 0.30029296875, -1.478515625, 1.40625, 0.53955078125, 0.7490234375, -0.4912109375, -0.166015625, 2.029296875, -0.1883544921875, 0.326171875, -0.90966796875, -1.1806640625, 0.01052093505859375, -0.0960693359375, 0.51513671875, -0.80615234375, 0.97216796875, -0.192138671875, -0.36767578125, -0.62451171875, 0.5341796875, -1.048828125, -2.08203125, -1.837890625, 0.251953125, -1.46484375, -0.294677734375, 1.4130859375, -0.1011962890625, -0.57958984375, 0.60595703125, 1.0146484375, -0.04888916015625, 0.23876953125, -0.2337646484375, 1.447265625, 2.3984375, -1.072265625, 0.828125, -1.228515625, -0.434326171875, -0.241455078125, -0.322509765625, -0.1375732421875, 0.52685546875, -0.061126708984375, -0.1370849609375, -0.96044921875, -2.783203125, -2.53515625, 0.045379638671875, 0.734375, -1.0341796875, -0.23095703125, 1.4033203125, 0.5419921875, -1.1845703125, 1.365234375, -0.73583984375, -0.75439453125, -1.5732421875, -1.02734375, -0.341552734375, 0.465087890625, -1.0390625, -0.0635986328125, 1.5185546875, 2.47265625, -0.0799560546875, -1.50390625, -0.53173828125, -0.416259765625, 1.28125, -0.046234130859375, -0.60009765625, 1.8837890625, -0.76416015625, -1.3642578125, 1.0087890625, -2.74609375, 2.076171875, -1.912109375, 1.2666015625, -0.982421875, 0.439697265625, 0.2841796875, 0.265869140625, -0.00647735595703125, 2.47265625, -0.96875, -1.4775390625, -1.4169921875, 0.52734375, -0.21337890625, -2.369140625, 0.52685546875, -2.65625, -1.013671875, 0.48046875, 0.6025390625, -0.87939453125, -1.1572265625, 0.80224609375, -1.21875, -0.8076171875, -1.4375, 0.2314453125, -0.406982421875, -0.4677734375, 0.30615234375, 1.0302734375, 0.451904296875, 0.02996826171875, -0.395751953125, 2.263671875, -1.046875, -0.88916015625, -0.26220703125, 0.059600830078125, 0.78564453125, -0.66943359375, -0.1700439453125, -1.31640625, -0.939453125, -0.99072265625, -0.56689453125, -0.309326171875, 0.8115234375, 1.064453125, 0.53564453125, 0.0360107421875, -0.2763671875, -0.68994140625, 0.6064453125, 0.46142578125, 0.65185546875, 0.62109375, -2.177734375, -0.28076171875, 0.4033203125, -0.52294921875, -0.642578125, -0.62158203125, -1.95703125, -0.2127685546875, -0.003749847412109375, 1.3720703125, -1.6005859375, -0.12255859375, -0.220703125, 0.345703125, -0.9072265625, 2.001953125, -0.51953125, -1.828125, -0.94921875, -0.20751953125, -0.99072265625, 0.10968017578125, -0.54345703125, -1.9248046875, 1.4599609375, -0.10357666015625, 0.96044921875, -0.2254638671875, -0.31298828125, 2.03515625, -0.236328125, 0.615234375, 0.4189453125, -1.1728515625, 1.826171875, 0.61083984375, 0.1795654296875, -0.327880859375, 2.59375, 1.0205078125, 0.186279296875, -0.286376953125, -2.525390625, -0.7548828125, -0.4892578125, 0.2279052734375, -0.09674072265625, 0.66650390625, 0.318359375, 0.2120361328125, -1.5537109375, 0.061859130859375, -0.5634765625, 0.86328125, 1.5390625, -1.6943359375, 0.03436279296875, 1.0322265625, 0.205810546875, -2.978515625, 0.59130859375, 0.3017578125, -0.9248046875, -0.1414794921875, 0.375244140625, -0.35888671875, 2.1171875, -1.2421875, 0.224609375, -1.0654296875, 0.2166748046875, 1.224609375, 0.278076171875, -1.3017578125, 1.515625, 1.2138671875, 0.841796875, -0.358642578125, 0.0673828125, -1.59765625, -0.53759765625, -1.5048828125, 0.424560546875, 0.1527099609375, -0.65234375, -0.1412353515625, -0.435546875, 0.64111328125, -0.231689453125, -0.60400390625, 0.8603515625, -0.14794921875, -0.00904083251953125, 0.2239990234375, -1.2958984375, -0.49609375, 0.65576171875, 0.92236328125, 1.5751953125, 0.60595703125, 0.8623046875, -0.31884765625, -0.5859375, 0.27685546875, -0.10845947265625, -0.8740234375, 1.6572265625, 1.283203125, 0.040618896484375, -0.2271728515625, 0.33203125, 1.70703125, -1.70703125, -1.37109375, 2.8203125, -1.484375, -1.3876953125, 0.2362060546875, 0.2734375, -1.3251953125, -1.7890625, 0.5126953125, -0.09649658203125, -0.31787109375, -1.4033203125, 0.66357421875, 0.07293701171875, 0.78466796875, 0.20361328125, -1.0126953125, -0.255859375, 0.55078125, 1.0029296875, -0.1427001953125, -0.80712890625, 0.947265625, -0.51806640625, 1.8583984375, 0.5517578125, 2.111328125, -0.11077880859375, 0.5927734375, 0.77783203125, -0.82275390625, 1.17578125, 1.23828125, -0.54248046875, -1.7373046875, 0.826171875, -0.01561737060546875, 1.115234375, -0.056488037109375, -0.88623046875, -0.325439453125, -0.46142578125, -1.6171875, -1.3505859375, -0.49560546875, -2.51953125, -0.470458984375, -0.83447265625, -0.66748046875, 1.4619140625, -0.329833984375, -0.094970703125, 1.3544921875, -1.6904296875, -0.211181640625, -0.1436767578125, -0.266357421875, -0.302490234375, 1.1298828125, -0.047210693359375, -0.195068359375, 0.285888671875, -2.00390625, 0.473876953125, -0.47314453125, 0.452880859375, -0.6669921875, 0.65380859375, 0.53466796875, -0.611328125, -1.169921875, 0.888671875, -2.65625, -1.115234375, 2.73046875, 0.309326171875, -0.91064453125, -1.234375, -0.1361083984375, -0.81640625, -0.85498046875, 0.447998046875, 0.546875, 0.78173828125, -0.59423828125, -1.2080078125, -1.0419921875, -0.498779296875, 1.6884765625, -0.61865234375, 3.734375, -0.9619140625, -0.417724609375, 1.1650390625, 0.61376953125, -0.302978515625, 0.079833984375, 0.387939453125, 0.703125, -0.170654296875, -2.04296875, 2.6015625, -0.82275390625, 0.69140625, -0.58447265625, -0.7060546875, 0.99658203125, -0.5888671875, -1.015625, -0.275390625, 0.378662109375, -0.12139892578125, 0.07171630859375, 1.33984375, -1.552734375, -0.406005859375, 0.6708984375, 0.293212890625, 1.005859375, -0.787109375, -0.708984375, -1.8212890625, 0.28271484375, 2.16796875, -2.267578125, 0.51318359375, -0.061004638671875, 0.8251953125, -0.5087890625, 0.049346923828125, 0.046173095703125, -1.041015625, 1.013671875, 0.1402587890625, 0.10980224609375, -0.60302734375, 1.2763671875, 1.1513671875, -0.7802734375, 0.2098388671875, 2.689453125, 1.7197265625, -0.9970703125, -0.58642578125, -0.3203125, -0.215087890625, 1.4794921875, 0.70947265625, -0.2073974609375, 1.142578125, -0.2337646484375, 0.3173828125, 0.5673828125, -0.489990234375, -1.2880859375, 0.10345458984375, -0.47314453125, -2.873046875, 0.38623046875, 1.1259765625, 0.222900390625, -0.62890625, -0.904296875, 0.55712890625, -1.5771484375, -0.107666015625, 1.1845703125, -2.251953125, -0.6591796875, 1.3525390625, -0.58056640625, 0.83837890625, 0.047515869140625, 0.07763671875, 0.94677734375, -0.74560546875, -1.4658203125, -1.4267578125, -0.176513671875, 0.44921875, -0.025390625, 0.0188446044921875, -0.1009521484375, 1.1982421875, -1.5, -1.185546875, -2.34375, -1.1650390625, 0.271240234375, 1.3603515625, -0.53271484375, 1.2548828125, -0.51806640625, 0.05548095703125, 1.638671875, -0.4970703125, 0.27197265625, 0.701171875, 0.7353515625, 0.86572265625, -0.35986328125, -0.489013671875, -0.32958984375, -0.20654296875, -1.52734375, 0.82763671875}; diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/data/data_64_64_64.h b/sw/vme/apps/tew16/fp16-matmul-vtse/data/data_64_64_64.h deleted file mode 100644 index 629ed44bd..000000000 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/data/data_64_64_64.h +++ /dev/null @@ -1,15 +0,0 @@ -// Auto-generated by tew16/fp16-matmul-vtse/script/gen_data.py. -#include "layer.h" - -static const vme_matmul_layer matmul_l = { - .M = 64, .N = 64, .K = 64, -}; - -static const float Cref_dram[4096] - __attribute__((section(".dram"), aligned(64))) = {-5.3203125f, -1.12890625f, -10.546875f, -2.80859375f, 13.5625f, -0.677734375f, -10.3046875f, -19.125f, -1.484375f, 11.921875f, -8.4765625f, 0.06317138671875f, -16.625f, -15.515625f, 9.4765625f, -19.421875f, -11.03125f, 7.25f, 3.6875f, 8.3046875f, 9.109375f, -2.654296875f, -4.7421875f, 6.34375f, -0.84326171875f, -1.2802734375f, -15.8359375f, 11.3125f, -2.8125f, -4.00390625f, 5.57421875f, -4.7578125f, -0.61474609375f, 12.421875f, 11.6953125f, 9.3984375f, -5.33203125f, 0.9345703125f, -3.3125f, -0.65771484375f, -5.046875f, -3.548828125f, -0.1632080078125f, 17.484375f, -2.2890625f, -12.15625f, -1.0791015625f, -7.98828125f, -3.5390625f, -17.015625f, 4.515625f, -11.125f, -4.02734375f, 1.9609375f, -4.4375f, -0.44189453125f, 2.892578125f, 1.7548828125f, -7.9921875f, -10.5546875f, -3.46484375f, -9.734375f, 8.28125f, 1.4833984375f, -5.4609375f, 4.25f, 0.013336181640625f, 7.60546875f, -8.2421875f, 6.04296875f, -12.1484375f, -5.40625f, -0.62353515625f, -5.77734375f, 0.53564453125f, 3.310546875f, -4.17578125f, -2.220703125f, -6.734375f, -9.6796875f, -15.640625f, -10.0703125f, 0.96728515625f, 19.390625f, -2.52734375f, 6.12109375f, -6.74609375f, 7.87890625f, -4.0625f, 1.759765625f, -5.88671875f, -1.19921875f, 7.74609375f, 3.0f, -1.9208984375f, -9.7734375f, -3.494140625f, 0.67529296875f, 8.890625f, 0.65234375f, -3.24609375f, -2.142578125f, 0.47802734375f, -11.71875f, -0.468505859375f, 2.986328125f, -6.5f, 8.9765625f, -1.3916015625f, 4.1796875f, 2.1484375f, -1.7177734375f, -0.90625f, 2.337890625f, 2.091796875f, -7.328125f, -9.1640625f, -2.0f, 16.078125f, 3.880859375f, 4.78515625f, -4.390625f, -6.8671875f, 0.9736328125f, -4.84765625f, -5.99609375f, 2.8046875f, -2.115234375f, 4.9140625f, 7.80078125f, -2.841796875f, -4.6953125f, 12.109375f, -7.71875f, 2.388671875f, -2.576171875f, 7.19921875f, -3.849609375f, 0.63720703125f, 1.2607421875f, 0.91552734375f, 3.451171875f, -6.1328125f, 1.4228515625f, 3.009765625f, -7.1875f, -4.92578125f, -8.71875f, 4.14453125f, -4.8203125f, 7.1796875f, -6.9921875f, 9.390625f, 4.21484375f, -3.619140625f, 7.0546875f, -0.384521484375f, 2.515625f, 4.05078125f, 4.73046875f, -1.2099609375f, 5.890625f, 4.359375f, 0.0221099853515625f, -12.53125f, -3.6484375f, 10.7109375f, 13.2890625f, 5.10546875f, -11.4296875f, 3.326171875f, 3.220703125f, -1.7978515625f, 6.91015625f, 3.875f, 13.734375f, -10.015625f, -1.240234375f, -8.5f, -1.443359375f, 14.8203125f, 11.15625f, -8.421875f, -5.5703125f, -6.2265625f, 7.9140625f, -3.634765625f, -6.75390625f, 8.7265625f, -1.4794921875f, -3.921875f, -3.712890625f, 8.2421875f, -18.09375f, 0.75830078125f, 0.26220703125f, 6.0234375f, -7.390625f, 2.63671875f, -19.125f, 1.7373046875f, 2.349609375f, -4.65234375f, -3.685546875f, -6.46875f, -14.7890625f, 20.265625f, 7.28125f, -3.349609375f, -0.2412109375f, -0.9814453125f, -2.96484375f, 3.537109375f, 0.935546875f, -19.296875f, 4.55859375f, 12.390625f, -5.90625f, -5.98046875f, -8.4375f, -9.015625f, 0.56787109375f, -1.5947265625f, 1.8701171875f, 9.8984375f, 5.6484375f, 4.9609375f, -3.16015625f, 2.091796875f, 18.5f, -8.3359375f, -13.296875f, 2.2109375f, 5.26171875f, 7.703125f, -3.29296875f, -13.65625f, 0.85400390625f, -3.12109375f, -2.927734375f, 5.92578125f, 1.4140625f, -3.86328125f, 0.413330078125f, 5.24609375f, 4.359375f, -13.453125f, -1.798828125f, -0.6484375f, 7.80859375f, -2.01953125f, -0.10223388671875f, 14.3359375f, -1.982421875f, 0.9052734375f, 10.3046875f, 8.8828125f, 11.25f, -11.7109375f, -5.484375f, -8.0859375f, 5.4765625f, -10.8359375f, 4.33203125f, -9.4375f, 5.921875f, -7.34375f, 5.42578125f, -10.2265625f, -16.1875f, -2.45703125f, -7.5234375f, -5.234375f, -3.90625f, 6.28125f, 10.625f, 12.9765625f, -3.513671875f, 3.271484375f, 14.5390625f, 0.74755859375f, 2.427734375f, -0.40087890625f, -0.736328125f, -11.7265625f, 15.1640625f, 5.2421875f, -8.5390625f, -11.1953125f, 0.69384765625f, -0.60107421875f, -9.328125f, -0.2470703125f, 0.4208984375f, -7.5390625f, -4.82421875f, -15.65625f, -14.296875f, -5.203125f, 3.587890625f, -6.078125f, -2.943359375f, 8.4609375f, -0.1748046875f, 8.3203125f, -1.0673828125f, 10.7578125f, 13.6171875f, -7.015625f, -3.53515625f, 19.484375f, 17.359375f, 9.7578125f, -15.4453125f, -0.96337890625f, -6.4453125f, 1.8642578125f, -6.70703125f, 4.0703125f, 1.013671875f, -2.1015625f, -6.3828125f, -5.34765625f, 5.76953125f, 4.5078125f, -9.1171875f, -3.109375f, -1.189453125f, -1.4775390625f, 1.4658203125f, 27.59375f, -6.8125f, -6.18359375f, 4.43359375f, 5.875f, 5.0703125f, -1.56640625f, -5.69140625f, 6.2578125f, 2.642578125f, 0.4921875f, -1.314453125f, 7.04296875f, -6.73046875f, -0.1884765625f, -14.4296875f, -5.1015625f, 4.6640625f, -7.92578125f, -5.43359375f, -1.3642578125f, -6.671875f, 3.38671875f, -11.8515625f, -7.453125f, -19.0625f, 4.390625f, 6.87109375f, -11.8359375f, -6.2421875f, 4.2109375f, 2.32421875f, 3.412109375f, 11.203125f, -2.595703125f, 4.125f, -6.296875f, 5.97265625f, -4.54296875f, -6.31640625f, 9.0625f, 2.966796875f, 11.6171875f, 8.796875f, -8.125f, -10.2109375f, -6.44921875f, 10.5859375f, 3.29296875f, -0.64599609375f, 5.80078125f, 6.34375f, -0.54296875f, 9.65625f, 6.30859375f, 0.61767578125f, 10.5f, 20.1875f, 11.015625f, -4.44921875f, 9.8203125f, -0.86865234375f, 8.1171875f, -0.9638671875f, 5.64453125f, -5.68359375f, 8.03125f, -14.0234375f, -8.3828125f, -4.2421875f, -1.3447265625f, 3.345703125f, -4.18359375f, -4.01953125f, 7.11328125f, -2.005859375f, -0.2298583984375f, -14.09375f, -0.904296875f, 1.3857421875f, 16.796875f, -3.58984375f, -4.125f, -1.26171875f, -4.23046875f, -10.7734375f, 10.8828125f, -6.5703125f, -9.1171875f, 2.46484375f, 12.1484375f, 4.76171875f, -11.0390625f, 8.3125f, 18.953125f, 4.92578125f, 13.7109375f, 5.19921875f, -5.0390625f, -7.5f, -2.017578125f, -9.9296875f, 7.3515625f, -3.765625f, -9.59375f, -10.140625f, 4.2578125f, 0.939453125f, -8.4921875f, 5.3671875f, -6.79296875f, 4.88671875f, 4.62109375f, -1.9326171875f, -9.6328125f, 11.2421875f, 9.625f, 7.7421875f, -0.2135009765625f, 0.01514434814453125f, -16.234375f, -13.671875f, 4.7109375f, -12.140625f, -2.251953125f, 5.02734375f, -1.21484375f, 6.515625f, 4.4609375f, 4.3203125f, -3.34375f, -0.44091796875f, -3.080078125f, 11.890625f, -1.8974609375f, -10.1796875f, 9.03125f, 0.10198974609375f, 3.01171875f, -3.76953125f, -3.013671875f, 3.5f, -6.984375f, -16.4375f, 4.8828125f, 20.359375f, -11.3125f, -4.47265625f, 9.5625f, -3.203125f, 6.3125f, -21.90625f, 0.0772705078125f, 16.1875f, 10.09375f, -14.859375f, -2.1640625f, -5.71875f, -9.765625f, -15.3828125f, -6.67578125f, -3.8671875f, -0.94580078125f, -6.78125f, 2.1796875f, 7.16015625f, -1.380859375f, 1.3232421875f, 10.1328125f, 8.359375f, 3.98828125f, -2.85546875f, 7.4765625f, -10.71875f, -0.9267578125f, 7.66015625f, 3.259765625f, -4.2421875f, 5.21484375f, -11.5625f, 10.8671875f, -6.33203125f, -17.65625f, -7.62109375f, -0.54638671875f, 4.87890625f, 2.64453125f, -6.046875f, -8.546875f, -0.041961669921875f, -0.489013671875f, 1.517578125f, -6.1953125f, 4.484375f, -1.072265625f, 0.431884765625f, -1.6416015625f, -1.5830078125f, 6.64453125f, -8.8671875f, -5.125f, -4.390625f, 1.966796875f, -2.373046875f, -2.08984375f, 11.6875f, -3.38671875f, 2.21875f, 6.7578125f, 1.642578125f, -3.71875f, -23.546875f, -4.234375f, -0.046478271484375f, -6.0234375f, -15.6875f, -7.85546875f, 14.1015625f, 12.390625f, 9.171875f, -3.767578125f, -5.10546875f, 6.0234375f, -5.2109375f, -0.822265625f, 4.1484375f, -1.6640625f, 3.666015625f, 8.6328125f, -7.1640625f, 1.9384765625f, 1.2734375f, -5.92578125f, 10.03125f, -8.0625f, -10.4375f, 4.65625f, -4.125f, 1.001953125f, -3.095703125f, -7.16796875f, -0.5166015625f, -0.33154296875f, -7.6015625f, -2.544921875f, 6.03515625f, -4.890625f, 0.469970703125f, 4.2265625f, -0.177490234375f, -14.0859375f, 6.0390625f, 1.1474609375f, 12.65625f, -6.8046875f, -10.3671875f, -6.875f, -16.21875f, 11.6328125f, 11.765625f, -18.484375f, -3.57421875f, 11.78125f, -8.359375f, -5.12890625f, -7.8828125f, -0.53662109375f, 11.3515625f, 19.0f, -3.697265625f, 4.203125f, -12.171875f, 2.865234375f, 13.8671875f, 4.359375f, -5.015625f, 8.2265625f, 1.0458984375f, -6.50390625f, -10.8203125f, 8.5546875f, -14.8359375f, 3.92578125f, -14.8203125f, 2.828125f, 12.015625f, 10.2890625f, 8.5078125f, 1.6953125f, 7.40625f, 4.02734375f, 14.4296875f, 2.58203125f, 3.3515625f, -0.83056640625f, 9.9765625f, 4.7109375f, 5.3125f, 11.78125f, -0.97412109375f, -8.8984375f, -15.78125f, -2.892578125f, -4.26171875f, -0.8388671875f, 7.05078125f, -10.453125f, 4.44921875f, -1.677734375f, -10.3984375f, -7.98828125f, 6.98828125f, 11.1171875f, 6.6015625f, 12.953125f, -6.77734375f, -1.6201171875f, -0.10577392578125f, 12.296875f, -8.046875f, 8.296875f, -4.9921875f, 7.66796875f, 1.267578125f, -6.84375f, -3.078125f, 5.015625f, 8.65625f, -0.95751953125f, -8.4375f, -5.03125f, 3.171875f, -0.2125244140625f, -1.4052734375f, 8.0859375f, 1.53125f, 16.625f, 5.72265625f, 3.0703125f, -9.546875f, 3.51171875f, -6.890625f, -3.037109375f, 3.744140625f, -3.625f, 5.54296875f, 2.71875f, -3.6640625f, 4.5859375f, 1.501953125f, 17.5625f, 9.2265625f, -6.44921875f, 5.78125f, -11.2890625f, -10.0703125f, -7.5078125f, 18.828125f, 5.22265625f, -2.28125f, -10.578125f, -13.625f, 2.802734375f, -8.2578125f, 5.8828125f, 1.7783203125f, 1.5966796875f, 8.0078125f, -0.5126953125f, -6.61328125f, 12.734375f, 0.755859375f, 2.609375f, -0.658203125f, 0.92529296875f, 6.9375f, 13.6171875f, 3.74609375f, -15.2890625f, -13.5859375f, 13.3046875f, -4.35546875f, -4.18359375f, -10.1015625f, -11.0859375f, -13.703125f, -2.876953125f, 4.2109375f, 1.515625f, -2.236328125f, 14.515625f, 6.88671875f, -6.4453125f, 10.953125f, 7.24609375f, 10.4453125f, 3.640625f, 12.0234375f, 8.578125f, 7.08984375f, -5.6015625f, 3.767578125f, -1.6005859375f, 3.6171875f, 0.24072265625f, 10.5234375f, -9.4453125f, 4.1484375f, 11.0234375f, -5.3515625f, -7.21875f, 22.875f, 4.921875f, -8.6484375f, 19.640625f, 13.4375f, -4.72265625f, 2.6328125f, 4.42578125f, 13.625f, -7.1796875f, 1.95703125f, -12.78125f, 21.953125f, -18.5f, 8.84375f, 5.0078125f, 11.5f, 3.30078125f, 0.54150390625f, -14.0f, 5.04296875f, 5.5390625f, 0.2088623046875f, 0.341064453125f, -0.4765625f, -1.6494140625f, -11.796875f, -7.34375f, 4.5f, 6.80859375f, 0.53955078125f, 2.81640625f, -5.75390625f, 20.875f, 10.6328125f, -0.611328125f, 6.4921875f, 11.0390625f, 9.46875f, 3.24609375f, -0.8095703125f, 3.060546875f, -6.4609375f, -0.08642578125f, 2.482421875f, -4.375f, 12.6171875f, 7.2890625f, -15.234375f, 9.6796875f, -13.3359375f, -2.333984375f, -6.859375f, 3.64453125f, 2.52734375f, -10.6953125f, 10.2109375f, -3.673828125f, -1.8349609375f, 13.2890625f, -13.6484375f, -3.908203125f, -16.09375f, 0.4248046875f, 4.69140625f, -10.40625f, 5.65234375f, -4.02734375f, -1.8642578125f, 6.78515625f, 9.9609375f, 4.046875f, 9.7265625f, 0.447998046875f, -4.99609375f, -2.775390625f, 6.96484375f, -5.203125f, 6.9765625f, 3.005859375f, 13.4921875f, -12.5703125f, -7.80078125f, -0.251708984375f, -5.109375f, -7.359375f, 1.140625f, 2.291015625f, 2.740234375f, -11.8359375f, -4.1953125f, 2.43359375f, -8.484375f, 3.375f, -11.9140625f, 0.400146484375f, 4.859375f, -1.208984375f, -17.4375f, -1.275390625f, -10.3984375f, -4.609375f, 3.6953125f, -1.869140625f, 11.546875f, 18.34375f, 6.24609375f, 2.978515625f, -8.296875f, 8.265625f, 3.6640625f, 5.15625f, 4.53515625f, -12.921875f, 9.625f, 9.2890625f, 0.397705078125f, -0.414794921875f, 6.8359375f, 18.796875f, -5.46484375f, 2.271484375f, 1.1064453125f, -9.421875f, 4.234375f, 3.626953125f, 3.181640625f, 2.341796875f, -5.3515625f, -2.083984375f, 1.8232421875f, -3.88671875f, 1.1171875f, 0.6005859375f, 2.48828125f, 2.462890625f, -1.28515625f, 3.5703125f, -3.0f, -3.431640625f, 1.5478515625f, -7.203125f, -10.2734375f, 1.75390625f, -2.66015625f, 2.607421875f, 4.3671875f, 5.375f, -11.4453125f, 1.84375f, 1.185546875f, 7.55859375f, -1.8134765625f, -1.0263671875f, 1.5556640625f, 8.625f, 11.2734375f, 9.453125f, 11.34375f, 5.2109375f, -2.826171875f, -4.3046875f, 0.982421875f, -1.736328125f, 7.8671875f, 9.21875f, 3.240234375f, 6.51953125f, -6.109375f, 1.828125f, -5.5703125f, -0.451904296875f, 13.3828125f, 10.703125f, 8.7421875f, -2.650390625f, -12.3984375f, 8.140625f, 18.71875f, 11.671875f, 7.078125f, 1.2373046875f, -5.04296875f, 9.1328125f, 2.14453125f, 8.78125f, -7.1640625f, -5.72265625f, 11.015625f, 6.89453125f, 2.5546875f, -5.328125f, -0.60400390625f, 1.8291015625f, 0.87939453125f, -7.5390625f, -6.49609375f, 14.1484375f, 12.859375f, -2.123046875f, -0.96728515625f, 4.05859375f, -3.619140625f, 4.515625f, 8.5625f, 4.81640625f, -11.3984375f, 3.31640625f, 11.625f, -2.68359375f, -5.43359375f, -3.08984375f, 5.9921875f, 1.1201171875f, 3.0703125f, 4.89453125f, -3.57421875f, 4.3359375f, -0.88623046875f, 11.6796875f, 11.2421875f, -2.22265625f, 18.25f, 6.6171875f, -3.5625f, 9.203125f, -10.7109375f, -5.546875f, -9.1640625f, -6.80078125f, -12.2890625f, -5.08984375f, -6.53125f, 1.005859375f, -2.01171875f, 7.21484375f, 20.765625f, -2.435546875f, 11.546875f, -5.51171875f, 5.015625f, -2.71875f, -1.8427734375f, 9.3203125f, 7.98828125f, -2.875f, 4.078125f, -0.495361328125f, -1.8916015625f, 1.3056640625f, 1.9970703125f, 7.80859375f, 4.20703125f, -11.7109375f, -5.296875f, -8.640625f, -5.19140625f, 11.625f, 5.3359375f, 14.265625f, 9.03125f, 10.921875f, -3.404296875f, -13.5546875f, -7.0859375f, -1.513671875f, 8.3359375f, 7.71875f, 9.8046875f, -3.82421875f, 4.984375f, 6.1953125f, 5.21875f, 0.34619140625f, 18.203125f, -14.7578125f, -1.962890625f, 1.8466796875f, -7.00390625f, 3.720703125f, 11.6953125f, 2.462890625f, 3.318359375f, -1.5439453125f, -1.7763671875f, -0.60595703125f, -0.03424072265625f, 15.6875f, -9.3203125f, -12.3125f, -6.140625f, 5.23828125f, 14.5703125f, 6.5703125f, -2.775390625f, -3.185546875f, 0.4033203125f, 14.1640625f, 4.69921875f, 4.44921875f, 8.734375f, -9.25f, 0.19482421875f, -10.3125f, -18.84375f, 13.234375f, -3.5234375f, 8.3125f, 15.8984375f, -11.375f, 4.5703125f, 3.759765625f, -3.0078125f, 10.8984375f, 3.25f, 2.390625f, 3.6328125f, 9.3046875f, 12.3125f, -6.71484375f, -4.46875f, 6.54296875f, 8.6484375f, 11.828125f, -12.34375f, 3.119140625f, -0.298828125f, 2.736328125f, -14.7109375f, 7.1796875f, 9.6875f, 3.42578125f, -0.86962890625f, -8.328125f, -3.568359375f, -0.91015625f, 9.2421875f, -3.498046875f, 8.3671875f, -0.7080078125f, -0.40625f, -7.05859375f, 7.29296875f, 4.41015625f, 12.7578125f, 10.4140625f, -6.359375f, 1.091796875f, 0.95556640625f, 4.7578125f, 4.57421875f, 0.55078125f, -7.56640625f, -8.875f, 8.75f, -0.88525390625f, -4.98828125f, 4.84765625f, 9.171875f, 6.65234375f, 0.974609375f, -18.796875f, -6.39453125f, -6.65234375f, 6.8515625f, -4.12890625f, -2.82421875f, -7.59375f, -3.083984375f, 17.859375f, -0.45751953125f, -1.6865234375f, 4.22265625f, 0.492919921875f, -0.66064453125f, 8.421875f, -8.5078125f, 10.5234375f, -12.265625f, -7.265625f, -3.953125f, -7.16015625f, 0.3310546875f, -7.16015625f, -12.3671875f, 0.5166015625f, 2.39453125f, -5.328125f, -4.0390625f, 1.890625f, 10.1796875f, -10.09375f, -11.6640625f, -3.900390625f, 3.556640625f, -1.1435546875f, 10.703125f, -1.0673828125f, -2.931640625f, 9.359375f, -0.48388671875f, 4.30859375f, -12.765625f, -1.4794921875f, 9.4609375f, 0.270263671875f, -10.25f, 14.828125f, -4.91796875f, 4.96875f, -3.474609375f, -4.47265625f, 11.78125f, 12.234375f, -3.31640625f, -10.390625f, -0.03076171875f, -6.953125f, 12.171875f, -16.109375f, -4.140625f, 5.859375f, 17.125f, -0.10736083984375f, 0.5390625f, -5.9609375f, 1.0888671875f, 2.427734375f, 8.78125f, 7.23046875f, 8.640625f, 4.39453125f, -4.7265625f, 14.6640625f, 0.74755859375f, 8.1953125f, -2.79296875f, 1.05859375f, 6.56640625f, 7.47265625f, 3.01171875f, 6.52734375f, 6.49609375f, -10.1484375f, 1.6884765625f, -6.30859375f, -5.40625f, -1.884765625f, 3.638671875f, 5.5546875f, -4.890625f, 9.46875f, 2.216796875f, -0.87353515625f, 9.3515625f, -7.9453125f, 1.6376953125f, 5.09765625f, -10.03125f, 4.37109375f, 15.0859375f, -4.56640625f, 5.42578125f, -11.5703125f, 5.16796875f, -0.12841796875f, -9.9765625f, -8.015625f, -12.515625f, 5.46484375f, -15.4375f, 1.125f, -10.84375f, -2.13671875f, 7.55859375f, 2.13671875f, -0.2030029296875f, 0.4951171875f, -8.5703125f, -1.5205078125f, 5.12109375f, 8.6171875f, 2.92578125f, -6.94921875f, 11.1328125f, 5.93359375f, -3.927734375f, -0.165283203125f, 13.3046875f, 10.6484375f, 9.8515625f, -2.94140625f, -4.703125f, 3.94921875f, -3.701171875f, -4.28125f, 2.44921875f, -1.4453125f, 5.44140625f, -5.48046875f, -6.01171875f, 1.8154296875f, 1.841796875f, -2.890625f, 1.73828125f, -4.54296875f, 6.4921875f, 0.9677734375f, -7.73046875f, 5.26171875f, 7.80859375f, 9.8515625f, -4.09375f, 1.0498046875f, 3.080078125f, -1.8017578125f, -8.328125f, 4.9140625f, 10.0078125f, 7.046875f, 2.2734375f, -5.87109375f, -15.6875f, -5.07421875f, 8.296875f, 5.9140625f, -4.2421875f, 7.20703125f, -8.40625f, 4.36328125f, 6.71875f, -8.90625f, -4.859375f, -4.0859375f, 13.5234375f, -10.1796875f, -7.08203125f, -0.1466064453125f, 1.87890625f, 3.14453125f, 1.306640625f, 1.7822265625f, -8.6875f, 10.1640625f, -3.3125f, -5.2109375f, -4.11328125f, -0.9970703125f, 4.9296875f, -5.09765625f, 13.234375f, 0.454833984375f, 4.17578125f, -15.2421875f, 5.78125f, -12.3125f, 14.015625f, -14.4140625f, 6.21484375f, -8.9375f, 11.203125f, 6.140625f, -10.359375f, -8.8828125f, 0.73291015625f, -11.8671875f, -4.9921875f, 4.32421875f, -6.7265625f, 6.546875f, 10.3359375f, -12.578125f, 6.6328125f, 1.4208984375f, 9.0703125f, -1.125f, 4.58203125f, 13.6640625f, -3.876953125f, 13.7890625f, -4.5390625f, -3.13671875f, -2.14453125f, -4.234375f, 3.9609375f, -0.1982421875f, 18.90625f, -6.7890625f, -7.65234375f, -7.2265625f, -8.390625f, 17.28125f, 18.1875f, -6.48046875f, -7.9375f, 13.0234375f, 9.625f, 1.580078125f, 9.515625f, -2.443359375f, -11.2421875f, -9.1875f, -10.7734375f, -1.4873046875f, -0.10467529296875f, 8.015625f, 7.74609375f, 2.001953125f, -1.5859375f, 21.671875f, 6.33984375f, -7.8203125f, 7.0625f, -18.796875f, 3.138671875f, 0.8271484375f, -5.08203125f, 1.8603515625f, -12.0625f, 14.5703125f, -2.66015625f, -8.78125f, -10.6953125f, 0.1484375f, 1.826171875f, -11.703125f, 6.609375f, 7.21484375f, -12.5f, 4.89453125f, 0.9580078125f, -18.453125f, -10.8046875f, -3.580078125f, -6.8203125f, 13.8828125f, -17.171875f, 9.21875f, 17.046875f, 18.96875f, -9.25f, -2.671875f, 16.3125f, -1.8203125f, -12.9765625f, 12.3046875f, -6.1015625f, 4.73828125f, 4.0078125f, 3.61328125f, -0.1220703125f, 13.3046875f, 0.3203125f, 7.82421875f, 8.8125f, 8.578125f, -12.78125f, -4.74609375f, 5.38671875f, 0.291748046875f, -6.078125f, -7.375f, 0.28125f, 11.3515625f, -15.671875f, 10.2421875f, 10.21875f, 3.767578125f, 10.5390625f, -1.896484375f, -13.3046875f, 10.375f, -5.6796875f, 1.767578125f, 3.87109375f, -3.2734375f, -4.83984375f, -0.252197265625f, -3.84375f, -1.89453125f, 2.34375f, -6.1875f, -0.0017261505126953125f, -1.67578125f, -4.33984375f, -2.171875f, -0.1153564453125f, -2.453125f, -11.484375f, 3.466796875f, -12.765625f, 6.0859375f, 16.0f, 2.205078125f, -4.33984375f, -6.02734375f, -1.0029296875f, 3.037109375f, 1.39453125f, -0.333251953125f, 1.083984375f, 3.28125f, 1.7978515625f, -4.859375f, 5.203125f, 6.421875f, -3.537109375f, 7.60546875f, 0.4384765625f, -7.51953125f, -0.5546875f, 6.51171875f, 6.72265625f, -13.0546875f, 12.953125f, -0.90087890625f, -5.6640625f, -8.40625f, 3.080078125f, 10.546875f, 4.29296875f, 5.3046875f, -0.82958984375f, 3.48828125f, 14.796875f, -0.8720703125f, 9.1640625f, 1.2373046875f, -4.90234375f, 0.755859375f, -8.015625f, -2.033203125f, -3.05078125f, -12.4921875f, -5.6171875f, -11.1640625f, -14.0234375f, -1.576171875f, -4.80859375f, -7.48046875f, -0.75048828125f, -9.28125f, -1.9111328125f, -2.955078125f, 18.40625f, 20.328125f, -3.2109375f, -9.0234375f, 6.68359375f, 2.771484375f, 2.087890625f, -6.46484375f, 1.978515625f, 11.4609375f, 8.15625f, 1.841796875f, 2.53515625f, -6.33203125f, -7.23046875f, 0.12066650390625f, -5.7578125f, 6.51171875f, 0.230712890625f, -11.125f, -2.208984375f, -13.34375f, 8.2265625f, -3.078125f, 9.28125f, -4.7734375f, 2.37109375f, 0.073974609375f, -3.91015625f, -4.9375f, 6.0390625f, -4.22265625f, 5.06640625f, 9.0859375f, -9.203125f, 4.32421875f, 6.46875f, 7.06640625f, -3.21875f, 0.2427978515625f, -0.96875f, -10.5859375f, -16.109375f, 2.416015625f, 9.40625f, 4.9609375f, -5.52734375f, 6.5546875f, -0.892578125f, 3.41796875f, -3.240234375f, -8.71875f, -6.84765625f, -3.013671875f, 5.6875f, -2.830078125f, -0.8193359375f, -5.51953125f, -7.45703125f, 9.3125f, -6.38671875f, -9.5546875f, -3.75390625f, 5.04296875f, 2.984375f, -6.09765625f, 2.005859375f, 7.9375f, -0.93212890625f, -4.89453125f, -5.671875f, -4.39453125f, 19.171875f, -1.365234375f, 11.09375f, -0.638671875f, -8.5625f, -2.20703125f, 8.8046875f, 0.373779296875f, -3.76953125f, 1.9384765625f, 10.25f, -19.015625f, -7.9453125f, -1.8974609375f, 5.08984375f, 7.16796875f, 10.53125f, -4.27734375f, 1.513671875f, 13.59375f, -2.955078125f, -2.048828125f, 0.438232421875f, -3.509765625f, 6.17578125f, 8.5625f, 14.09375f, -7.06640625f, 0.051483154296875f, -6.16015625f, -1.21875f, -1.1123046875f, -0.267578125f, -0.1697998046875f, -4.65625f, 1.2216796875f, -9.875f, 1.431640625f, 3.57421875f, 3.994140625f, -1.6083984375f, -8.9453125f, 9.6640625f, 3.029296875f, 1.3291015625f, -7.6796875f, 1.537109375f, -12.7265625f, -4.28125f, 9.578125f, -2.607421875f, -8.140625f, 2.388671875f, 3.79296875f, 3.51953125f, -0.48583984375f, -9.1640625f, -2.68359375f, 0.681640625f, 9.59375f, 18.453125f, 4.8046875f, 5.59765625f, 2.2421875f, -3.212890625f, -11.7109375f, -6.37109375f, 1.6416015625f, -6.16015625f, -4.47265625f, 6.46484375f, 3.306640625f, -5.86328125f, 8.6484375f, -0.287353515625f, 1.384765625f, 12.8828125f, -1.23828125f, -8.78125f, 13.0234375f, 12.1875f, -15.8984375f, -3.056640625f, 3.015625f, -6.53125f, -4.78515625f, -3.5234375f, -11.5390625f, 2.6640625f, -0.99560546875f, 7.875f, 9.96875f, -3.01953125f, 1.603515625f, 15.4296875f, -2.5078125f, 4.55078125f, -0.53076171875f, 0.73583984375f, -0.87109375f, 7.95703125f, -0.08966064453125f, -4.109375f, 1.673828125f, 15.6484375f, 0.0489501953125f, -6.56640625f, 5.0078125f, -7.52734375f, -8.5f, 8.6875f, -8.4296875f, -10.0234375f, -8.1328125f, -17.875f, -2.7890625f, 3.306640625f, -6.671875f, 3.3828125f, -11.953125f, 9.34375f, 5.33203125f, 12.890625f, -13.65625f, 11.4296875f, -3.7421875f, 14.5078125f, 7.78125f, 6.8359375f, -1.611328125f, -3.40234375f, -3.75f, 3.501953125f, -21.640625f, -0.93212890625f, 5.62109375f, 6.56640625f, -2.96484375f, 10.8515625f, 3.447265625f, 6.91796875f, 14.125f, 0.6357421875f, 1.0263671875f, -1.5078125f, 4.34375f, 5.9375f, 5.15234375f, 8.7109375f, 4.453125f, 1.1884765625f, 8.1484375f, 8.1484375f, -2.734375f, -5.56640625f, 6.29296875f, 1.794921875f, 3.208984375f, -8.5078125f, 1.5849609375f, 16.375f, 1.7705078125f, 3.537109375f, -9.5859375f, -10.234375f, 11.59375f, -7.015625f, -8.6484375f, 2.318359375f, -8.6796875f, 4.125f, 2.068359375f, -1.08984375f, 7.94921875f, -5.36328125f, -4.00390625f, -4.5703125f, -0.693359375f, -1.5732421875f, -3.443359375f, -2.064453125f, -1.6953125f, -10.6796875f, 3.123046875f, -7.58203125f, 1.6396484375f, 1.173828125f, -6.8984375f, -10.7578125f, -1.455078125f, -2.98828125f, 2.3671875f, -3.82421875f, -1.62109375f, -23.0625f, 1.1806640625f, -0.99853515625f, -13.71875f, 2.880859375f, -4.1328125f, -0.94482421875f, 7.95703125f, -8.640625f, 4.4296875f, -7.01953125f, -1.5673828125f, 10.875f, 2.384765625f, -8.5390625f, 6.1875f, -10.6171875f, 4.99609375f, -3.017578125f, -4.7734375f, -11.796875f, -2.935546875f, -7.00390625f, -11.3515625f, -4.83203125f, -10.203125f, 0.900390625f, -10.828125f, -2.25390625f, 4.09375f, -7.19140625f, -9.8203125f, 3.84375f, -1.7744140625f, 7.140625f, -1.8564453125f, 1.4169921875f, 2.716796875f, 4.05859375f, 14.6171875f, -0.900390625f, -2.240234375f, 3.587890625f, -2.11328125f, -2.162109375f, -8.875f, 7.265625f, -6.2890625f, 2.072265625f, -17.640625f, 7.96484375f, -3.0390625f, 0.378662109375f, 7.890625f, 3.244140625f, 13.8984375f, 6.87109375f, -2.30859375f, -7.33203125f, -3.609375f, 5.9140625f, 8.0390625f, -1.91796875f, -0.4833984375f, 7.42578125f, 12.203125f, -8.1953125f, -8.6796875f, -2.51171875f, -4.484375f, -4.20703125f, -0.7822265625f, -0.1014404296875f, 10.015625f, -12.0078125f, -7.2890625f, -3.67578125f, -7.3125f, -1.6923828125f, 6.69921875f, -0.27783203125f, -9.671875f, -1.4482421875f, -0.0308380126953125f, 14.0234375f, 6.546875f, 7.640625f, 3.20703125f, 4.17578125f, -13.2734375f, -3.837890625f, -2.64453125f, 5.57421875f, -7.50390625f, -10.3984375f, -21.5f, -6.07421875f, -1.732421875f, 9.8671875f, -19.796875f, 4.12890625f, 6.3984375f, 3.453125f, -1.9794921875f, -7.62109375f, 4.578125f, 19.65625f, -0.88330078125f, -9.53125f, -3.505859375f, -4.15234375f, 12.765625f, 2.54296875f, 6.21875f, -17.0f, 3.287109375f, 1.5263671875f, 2.904296875f, -19.390625f, -16.3125f, 6.5234375f, 8.140625f, -16.046875f, 0.6435546875f, 11.65625f, 5.21484375f, 18.875f, -11.2578125f, -4.234375f, -10.03125f, 3.80078125f, -3.595703125f, -0.93359375f, -3.314453125f, 11.28125f, -9.4921875f, -12.515625f, 1.677734375f, 5.61328125f, 8.9453125f, 9.1328125f, -7.3046875f, -11.5625f, 14.953125f, -20.625f, -1.6513671875f, -12.921875f, -2.486328125f, -20.265625f, 5.11328125f, -6.66015625f, 4.04296875f, 19.6875f, -10.28125f, -4.18359375f, -2.537109375f, 9.234375f, 8.5078125f, 4.21875f, 15.375f, 4.84765625f, 9.859375f, 0.326416015625f, 3.53125f, 0.2318115234375f, -1.3232421875f, 3.7265625f, -2.87890625f, -1.89453125f, 8.71875f, -5.30859375f, -3.189453125f, -3.951171875f, -1.578125f, -6.64453125f, 5.421875f, -5.36328125f, 9.2578125f, 9.140625f, 12.421875f, 0.425048828125f, -8.203125f, 13.0390625f, 13.0390625f, 1.4921875f, -1.119140625f, 3.62109375f, 13.7421875f, 0.011383056640625f, -7.3984375f, -5.9375f, -7.0546875f, 4.22265625f, -8.6640625f, -6.9921875f, 1.015625f, 2.95703125f, -0.5107421875f, -3.32421875f, 6.12109375f, -5.203125f, 9.65625f, -0.19580078125f, -3.94140625f, 11.0234375f, 6.7421875f, -0.5830078125f, -11.40625f, -5.12109375f, -7.59765625f, 5.0078125f, -2.66796875f, 8.25f, 1.2431640625f, -7.70703125f, -11.1328125f, -13.8125f, 4.96875f, -2.732421875f, 7.9765625f, -11.4140625f, -2.70703125f, 0.25f, 1.9970703125f, -12.8046875f, 6.453125f, 6.9453125f, 2.015625f, 2.8984375f, -9.84375f, 10.125f, 4.3515625f, -1.8037109375f, 1.8759765625f, 12.78125f, -2.4921875f, -4.89453125f, -9.4140625f, -1.6337890625f, 13.125f, 11.5234375f, 2.953125f, 4.08984375f, 2.9765625f, 2.87890625f, -3.78125f, -7.0f, -12.875f, 2.1875f, 6.57421875f, 0.4453125f, 0.65478515625f, 0.32763671875f, 14.140625f, 2.708984375f, 5.46484375f, 7.23828125f, -9.5546875f, -8.9765625f, 3.8203125f, 4.234375f, 1.5048828125f, 13.4609375f, -0.69189453125f, 1.419921875f, 3.4140625f, 3.259765625f, -2.53125f, -10.7734375f, 4.046875f, -13.2578125f, 13.3125f, -9.890625f, -4.54296875f, -11.2109375f, -7.890625f, -3.568359375f, 1.61328125f, 0.88671875f, 0.43603515625f, -4.29296875f, -9.015625f, -13.828125f, 7.28515625f, 1.0087890625f, -11.203125f, -8.7265625f, -7.7109375f, -6.55078125f, 6.21875f, -3.73828125f, 7.12109375f, -7.359375f, -6.07421875f, 3.1484375f, 0.1939697265625f, 4.8828125f, -17.84375f, -11.2109375f, -10.2890625f, -11.0390625f, -2.18359375f, -4.73046875f, 0.552734375f, 8.265625f, -0.0748291015625f, -2.9765625f, 0.238037109375f, 2.962890625f, 2.08984375f, 7.453125f, 3.978515625f, 6.609375f, -12.25f, -5.00390625f, 3.123046875f, 9.0390625f, 12.65625f, -5.625f, -8.3046875f, -13.40625f, 1.181640625f, -20.375f, 6.72265625f, -7.26171875f, -6.22265625f, 10.046875f, -3.3671875f, -1.3203125f, 13.6015625f, -11.0390625f, -2.609375f, 1.3779296875f, -6.2734375f, 4.546875f, -16.078125f, 8.703125f, 0.318359375f, -5.26953125f, -4.33203125f, 15.4609375f, 0.097900390625f, -0.1126708984375f, -5.21484375f, 9.265625f, 1.48046875f, -9.6328125f, -2.193359375f, -0.673828125f, -1.0712890625f, -6.44921875f, -9.40625f, -12.1875f, -7.95703125f, 16.21875f, -12.828125f, 3.3671875f, 1.037109375f, -10.4375f, -0.595703125f, -6.15625f, 13.1015625f, -3.4375f, 2.009765625f, 10.828125f, -3.55078125f, -5.03515625f, 0.77783203125f, -9.8203125f, 3.40234375f, -13.1171875f, 5.11328125f, 11.7109375f, -20.578125f, 3.310546875f, 6.86328125f, -4.0859375f, -4.3515625f, 7.53515625f, 0.64208984375f, 1.162109375f, 4.6171875f, 1.220703125f, -4.515625f, -4.53125f, -0.6181640625f, -6.1875f, -3.0546875f, -4.15234375f, 0.9375f, -9.171875f, 12.1015625f, -2.4296875f, -10.8515625f, 0.160888671875f, -7.37890625f, 0.2294921875f, 0.2646484375f, 12.2890625f, -5.78125f, 3.771484375f, 2.302734375f, 8.0390625f, 3.3984375f, -6.98046875f, -7.6640625f, 2.498046875f, -7.37109375f, 2.1640625f, -6.41796875f, 22.796875f, -1.5673828125f, 2.310546875f, 9.8671875f, 4.921875f, 12.46875f, -7.25f, -10.828125f, 0.46875f, 2.30859375f, -7.82421875f, 6.03515625f, -9.625f, -4.36328125f, 6.22265625f, -6.4140625f, 6.70703125f, 5.33984375f, 5.55859375f, 25.09375f, 2.328125f, 4.8046875f, 10.9296875f, 5.6171875f, 9.9453125f, 11.703125f, 1.5419921875f, 3.462890625f, -5.6640625f, -11.4609375f, -2.474609375f, 1.669921875f, 3.17578125f, 1.3271484375f, -6.84765625f, -0.363525390625f, -7.9609375f, -1.701171875f, -14.015625f, -3.482421875f, 3.326171875f, 6.87109375f, -1.2568359375f, -17.484375f, -1.2080078125f, -11.8203125f, -2.767578125f, 14.953125f, -3.408203125f, -10.0f, 2.228515625f, 12.703125f, -8.4296875f, -2.427734375f, 9.5546875f, 1.677734375f, -6.8671875f, 6.7109375f, -10.6328125f, -14.359375f, -7.25f, 1.0107421875f, 2.83203125f, 6.96875f, -14.28125f, -5.5859375f, -6.73046875f, 3.677734375f, -3.1171875f, -10.796875f, 3.89453125f, -8.140625f, 2.654296875f, -10.9921875f, -7.28515625f, -2.97265625f, -12.546875f, -1.34765625f, 6.0703125f, -10.6796875f, 3.947265625f, 8.828125f, 2.25f, 5.74609375f, -3.10546875f, -9.8515625f, -3.658203125f, 1.904296875f, -1.2158203125f, -7.9140625f, 13.203125f, -2.634765625f, -1.7841796875f, 1.240234375f, 2.091796875f, -5.55078125f, -9.515625f, -12.0546875f, -0.256591796875f, 3.08203125f, -9.1171875f, 1.4765625f, 17.359375f, 7.55859375f, 3.734375f, 2.666015625f, -0.11138916015625f, 3.87109375f, 1.09375f, 4.1328125f, 14.7734375f, -16.28125f, -8.8671875f, 2.2734375f, -1.3564453125f, -1.6044921875f, 16.234375f, 5.07421875f, -1.6171875f, 1.3232421875f, 3.1484375f, -7.71484375f, 10.296875f, 5.5546875f, 3.48828125f, -1.1435546875f, 2.048828125f, -15.2109375f, 20.484375f, -1.9697265625f, -8.109375f, -7.390625f, 0.21533203125f, 10.875f, -8.34375f, -5.90625f, -3.072265625f, 5.578125f, -0.410888671875f, -1.576171875f, -12.9765625f, -14.640625f, 2.244140625f, 0.79443359375f, -11.46875f, 5.9453125f, 1.7021484375f, -6.17578125f, -7.22265625f, -8.53125f, 0.006565093994140625f, -12.5625f, 0.98095703125f, -9.75f, -2.294921875f, -0.330322265625f, 8.65625f, 5.6328125f, 2.46875f, -8.8359375f, 9.3515625f, 2.79296875f, 8.765625f, -12.1796875f, 9.859375f, 3.490234375f, 4.09375f, 15.5859375f, -1.9482421875f, -1.125f, -9.8984375f, 22.328125f, -2.095703125f, 2.09375f, 7.08984375f, -9.671875f, 7.89453125f, -1.40234375f, 6.1953125f, -4.96484375f, 5.734375f, 10.1875f, -6.9375f, 0.66064453125f, 9.6796875f, 9.2890625f, 4.57421875f, -5.5f, 4.67578125f, 0.35009765625f, -6.63671875f, 10.4453125f, 2.43359375f, -3.708984375f, -7.171875f, 1.1533203125f, -3.765625f, -7.625f, 5.74609375f, 8.046875f, -1.033203125f, 3.890625f, -10.34375f, 12.6328125f, -11.359375f, 6.57421875f, -8.40625f, 1.3271484375f, -6.6953125f, 2.966796875f, -1.3125f, -9.4765625f, -2.625f, -7.51171875f, 2.31640625f, -2.095703125f, -8.7734375f, 1.79296875f, -2.794921875f, 4.2734375f, 3.3046875f, -14.171875f, -0.986328125f, 2.736328125f, 2.14453125f, -0.9609375f, 1.2236328125f, -12.546875f, 7.6171875f, 4.73046875f, -5.54296875f, 2.8515625f, -4.48046875f, 0.66064453125f, -2.193359375f, 7.40234375f, -9.609375f, 10.0625f, 5.46875f, -11.0625f, -8.421875f, 3.177734375f, -10.6640625f, -7.95703125f, 5.09375f, 2.001953125f, 5.34375f, 4.9375f, 14.5859375f, 16.71875f, 3.822265625f, 0.259765625f, 13.3046875f, -7.41015625f, 0.47412109375f, -3.416015625f, 15.3828125f, -10.046875f, -4.65234375f, -2.08203125f, 7.15625f, 1.3837890625f, -16.390625f, 1.380859375f, 6.44140625f, 3.53515625f, 2.84375f, -4.26953125f, -5.2421875f, 0.1318359375f, 3.609375f, -0.11932373046875f, -4.58984375f, 5.23828125f, -6.8359375f, -12.96875f, -6.57421875f, -10.59375f, 2.59375f, 5.33203125f, -2.58984375f, 1.970703125f, -2.263671875f, 11.5078125f, 6.63671875f, 10.9140625f, -2.802734375f, 7.16015625f, 2.064453125f, 0.89208984375f, -9.25f, 4.28515625f, -16.40625f, 3.591796875f, 5.96875f, 11.109375f, -4.12890625f, 11.0078125f, 4.97265625f, -0.3466796875f, 0.89892578125f, 7.34765625f, 2.857421875f, -13.5f, -2.494140625f, -1.357421875f, -6.92578125f, -11.53125f, 1.3046875f, -7.359375f, 4.98828125f, 5.51953125f, 8.0546875f, 19.765625f, 10.25f, -7.484375f, 5.984375f, -2.4140625f, -15.984375f, 7.51171875f, 10.4140625f, 5.0859375f, -6.9765625f, 12.7265625f, 5.37109375f, -16.234375f, -15.71875f, -7.94921875f, -3.939453125f, 9.265625f, -10.3671875f, 7.5859375f, 3.15234375f, -1.7373046875f, 11.25f, 13.28125f, 4.03515625f, -2.890625f, -9.5859375f, -2.6875f, 1.3427734375f, -0.73095703125f, -0.6748046875f, -5.62109375f, 14.6015625f, -10.171875f, 6.7265625f, -9.3359375f, -8.15625f, -1.2646484375f, -7.89453125f, -7.4296875f, 4.671875f, 3.609375f, -2.25390625f, 5.71484375f, -4.33984375f, -4.9765625f, 14.6484375f, -6.80078125f, 0.701171875f, -1.7236328125f, -2.62890625f, 0.6552734375f, -3.41796875f, 3.599609375f, 9.65625f, -10.4765625f, -4.20703125f, 3.990234375f, 6.87109375f, 5.1953125f, -4.26171875f, 7.48046875f, 2.25f, -3.37109375f, -13.8359375f, -11.921875f, -4.375f, 3.44140625f, 3.72265625f, 0.7978515625f, -7.01171875f, -10.2421875f, -14.4453125f, 3.486328125f, -0.47509765625f, -3.880859375f, -13.84375f, -5.1875f, -0.1143798828125f, -2.23046875f, 1.509765625f, -4.28515625f, 5.79296875f, -7.55859375f, -8.8515625f, 11.5390625f, 8.9765625f, 0.357666015625f, 0.25341796875f, 9.453125f, 4.7109375f, 9.2890625f, -2.7109375f, 11.6484375f, -7.62109375f, 6.36328125f, 9.0078125f, 7.76953125f, 15.8125f, 1.560546875f, -8.6796875f, -3.0078125f, 3.875f, 8.5078125f, 11.9609375f, 3.736328125f, -1.7646484375f, -2.529296875f, -1.2529296875f, -11.9140625f, -0.2010498046875f, -6.1640625f, 0.460205078125f, 5.7578125f, -1.28515625f, 2.294921875f, -17.0625f, -6.93359375f, -1.2490234375f, -1.697265625f, -3.109375f, -13.109375f, 17.46875f, 13.0546875f, 1.89453125f, -2.109375f, -9.984375f, 14.1640625f, -17.53125f, 2.001953125f, -13.6328125f, -2.998046875f, -16.1875f, -12.2109375f, 2.828125f, 2.220703125f, 3.21875f, -0.024322509765625f, -7.6484375f, -8.015625f, -13.4375f, 6.5f, 4.60546875f, -6.10546875f, 1.5732421875f, 14.1796875f, 3.57421875f, 8.2109375f, -7.4140625f, 7.26953125f, 24.0625f, -4.1953125f, -0.341796875f, -14.125f, 10.3671875f, -28.25f, 20.421875f, -4.81640625f, -15.296875f, 2.8359375f, 9.375f, 2.1796875f, -8.2890625f, -5.9375f, -18.09375f, -14.9375f, -12.703125f, 8.515625f, 7.453125f, 12.5234375f, 5.3359375f, -1.46484375f, 8.6328125f, -4.1328125f, 11.015625f, -7.04296875f, -7.00390625f, -0.02777099609375f, -1.61328125f, -2.609375f, -5.875f, -0.1376953125f, -9.578125f, -11.5234375f, 16.640625f, -0.80078125f, -8.9296875f, 13.2265625f, 11.875f, -11.1796875f, -12.5625f, -5.671875f, 6.1015625f, -4.82421875f, 4.390625f, -7.87890625f, 10.9296875f, 3.857421875f, -18.9375f, -7.87109375f, -0.8056640625f, 10.46875f, 15.984375f, -10.0390625f, 3.1875f, 4.66796875f, 4.9609375f, 5.88671875f, -11.6015625f, -6.47265625f, -1.927734375f, -25.234375f, 18.140625f, 0.047027587890625f, 3.1171875f, -10.9375f, -7.23828125f, -2.123046875f, -8.2265625f, -27.65625f, -1.6728515625f, -4.80078125f, 10.515625f, -8.625f, -7.3671875f, 4.4765625f, -11.1796875f, 12.6796875f, 2.287109375f, 5.94921875f, 10.3046875f, -7.92578125f, -4.24609375f, 11.6484375f, 5.8515625f, 3.806640625f, -1.7666015625f, 4.21875f, -9.9609375f, 6.203125f, -7.38671875f, -3.375f, -9.7265625f, 13.0546875f, 8.2109375f, 0.1702880859375f, -3.4609375f, -1.38671875f, -0.057647705078125f, 6.43359375f, 3.943359375f, -11.0234375f, -4.26953125f, 2.55078125f, -0.252685546875f, 2.3828125f, 4.96484375f, 2.44140625f, -3.388671875f, 7.35546875f, -5.7734375f, -11.625f, -7.9453125f, -0.82080078125f, -1.5517578125f, -3.8671875f, 3.66796875f, -4.5f, 6.3125f, -15.828125f, -6.94140625f, -7.84375f, 2.376953125f, 2.515625f, -1.9013671875f, -11.0078125f, 9.1875f, 5.0234375f, 1.33984375f, -4.109375f, -13.171875f, -6.296875f, 13.9375f, -10.421875f, -4.609375f, -1.5791015625f, -12.328125f, 2.783203125f, -12.046875f, 1.005859375f, -6.375f, -6.44140625f, -0.8134765625f, 15.6171875f, 5.69921875f, -17.90625f, -6.9140625f, 3.951171875f, 12.484375f, 11.6015625f, -2.279296875f, -6.14453125f, -21.328125f, 17.125f, -10.796875f, -1.103515625f, -3.0703125f, 3.240234375f, 15.1875f, 1.9248046875f, -0.042694091796875f, 7.171875f, 2.38671875f, -3.869140625f, 6.01953125f, -11.96875f, 14.5546875f, 2.525390625f, -11.625f, -7.5625f, 2.2578125f, -3.220703125f, 0.020751953125f, 5.38671875f, 0.22900390625f, -1.4052734375f, 0.50390625f, 5.2421875f, 2.71875f, 3.650390625f, 9.7578125f, -4.46875f, -4.59375f, -8.9765625f, 4.41015625f, -6.51953125f, 3.03125f, -5.00390625f, -5.20703125f, 6.0859375f, -10.5078125f, 1.7099609375f, -10.2734375f, 2.5859375f, 8.1171875f, -6.74609375f, -14.2265625f, 0.28466796875f, -1.3388671875f, 15.3125f, 8.4140625f, 11.390625f, 7.5f, 0.4892578125f, -4.21875f, -2.3359375f, -9.90625f, 1.8916015625f, 2.611328125f, 13.6640625f, -3.955078125f, -4.95703125f, -7.3671875f, -1.5400390625f, 16.703125f, -9.7265625f, 0.0002300739288330078f, -15.171875f, 3.72265625f, -0.59814453125f, 14.0234375f, -7.265625f, 13.1015625f, -11.4140625f, 4.015625f, -0.76611328125f, 0.81982421875f, 19.671875f, 7.9765625f, -4.83203125f, -9.6171875f, -1.275390625f, 1.1259765625f, 6.328125f, -9.5234375f, -0.89990234375f, -5.43359375f, 5.65625f, -2.625f, 1.2109375f, 6.91796875f, -3.544921875f, -5.32421875f, 4.8515625f, 6.02734375f, -12.796875f, -8.9609375f, 15.734375f, 4.296875f, -3.98828125f, -9.5546875f, -2.369140625f, 6.0390625f, -10.7734375f, 7.54296875f, -0.5771484375f, 13.09375f, 12.65625f, 15.3984375f, -10.1171875f, -5.14453125f, 2.134765625f, 7.9140625f, -2.732421875f, -6.82421875f, -1.58984375f, 1.1220703125f, 3.193359375f, -3.6640625f, -1.0166015625f, -2.390625f, 2.134765625f, 11.6640625f, -3.921875f, -0.78125f, -14.0f, 7.9765625f, 9.4375f, 5.453125f, -12.3046875f, 6.40234375f, -15.0546875f, 13.1875f, -10.28125f, -1.142578125f, -11.3984375f, 10.34375f, 2.169921875f, 4.39453125f, 3.029296875f, -10.09375f, 1.48046875f, 7.2265625f, -0.2379150390625f, 3.61328125f, -5.14453125f, -4.74609375f, -3.58984375f, 4.69921875f, -2.873046875f, -3.060546875f, 9.2890625f, -3.267578125f, -2.345703125f, -4.94140625f, -4.18359375f, -0.487060546875f, -1.802734375f, -9.5859375f, -17.921875f, -10.96875f, -0.71142578125f, 5.5625f, 1.9970703125f, -3.912109375f, 3.5234375f, 6.734375f, -15.578125f, 2.005859375f, -13.1484375f, -0.83203125f, -11.5f, 0.33349609375f, 5.375f, 2.54296875f, 1.271484375f, 7.0859375f, -7.05078125f, -16.65625f, -4.61328125f, -1.9736328125f, -3.662109375f, 8.2421875f, -5.26953125f, -10.875f, -1.7158203125f, -4.9453125f, -7.48046875f, -3.01171875f, -7.5078125f, 8.2578125f, 7.98046875f, 17.9375f, 3.224609375f, 7.80859375f, 0.95751953125f, 3.037109375f, -18.578125f, -1.51953125f, -0.79736328125f, -3.990234375f, -5.140625f, -6.12890625f, 0.5244140625f, 10.2421875f, 15.765625f, 2.28125f, -4.890625f, -4.41015625f, 14.21875f, -1.62109375f, -4.92578125f, 7.07421875f, 2.81640625f, 0.693359375f, 12.125f, -7.03125f, -3.708984375f, 5.515625f, -7.94921875f, 0.96484375f, 0.5341796875f, -7.45703125f, -4.5625f, 1.947265625f, 8.953125f, -3.7265625f, 1.4951171875f, -1.4453125f, -5.16796875f, -6.09765625f, 12.109375f, -4.453125f, -0.72021484375f, 5.46484375f, 9.4921875f, -6.4140625f, -3.02734375f, -4.5546875f, -10.5390625f, -0.291015625f, -8.1328125f, -5.36328125f, -11.1015625f, -8.0078125f, -6.1015625f, 0.55224609375f, 6.69921875f, 22.328125f, -4.51171875f, 12.8359375f, 2.232421875f, -0.73779296875f, -5.24609375f, -3.23828125f, -1.03515625f, 5.6328125f, 12.15625f, 18.78125f, -11.9375f, 16.65625f, -5.33984375f, -4.4296875f, 17.140625f, -12.5234375f, -3.09765625f, 2.763671875f, -25.828125f, 2.392578125f, 3.46484375f, -6.9140625f, -0.1907958984375f, -3.228515625f, 4.29296875f, -2.21875f, 6.06640625f, 10.609375f, -10.4140625f, -1.2177734375f, -9.171875f, 10.1328125f, -2.0703125f, -2.625f, 1.4306640625f, 8.6640625f, 5.65625f, 9.03125f, 3.88671875f, 2.302734375f, 19.296875f, 3.94921875f, 0.8232421875f, -9.3984375f, 5.5390625f, -0.9658203125f, 6.04296875f, 6.1328125f, -8.828125f, 5.2109375f, 6.3125f, 0.59765625f, -5.80859375f, -10.640625f, 14.140625f, -2.55078125f, -1.8984375f, -24.4375f, -3.24609375f, -1.8046875f, 13.9453125f, 9.609375f, -1.9931640625f, 1.546875f, -0.37841796875f, 24.25f, 0.68603515625f, 0.63671875f, 3.9375f, 11.8203125f, 5.5078125f, 2.884765625f, 5.1953125f, -5.73828125f, 1.439453125f, -1.810546875f, -14.0390625f, 13.6640625f, 7.0546875f, -1.4873046875f, -0.15087890625f, 12.1171875f, -7.71875f, 11.265625f, -8.6328125f, -3.908203125f, 3.392578125f, 8.3515625f, 6.78125f, 6.421875f, 6.265625f, -12.21875f, 2.72265625f, -4.12109375f, -5.6875f, 6.71875f, -6.1796875f, 22.34375f, 1.8857421875f, 20.671875f, 4.30859375f, -2.32421875f, -2.02734375f, 12.328125f, 22.671875f, -0.0204620361328125f, 0.63818359375f, -4.75f, 6.15625f, 15.7265625f, 8.671875f, 3.48046875f, -4.36328125f, 2.453125f, 3.298828125f, -1.880859375f, -14.609375f, -4.33984375f, 1.3564453125f, -8.4375f, -11.78125f, -22.078125f, -2.0234375f, -0.1400146484375f, -1.8310546875f, 7.27734375f, -5.06640625f, 5.1015625f, -1.181640625f, 6.7734375f, 1.0224609375f, 5.34765625f, -11.5f, 4.203125f, 1.28125f, 0.7216796875f, 6.3046875f, 19.546875f, -4.734375f, 10.765625f, -8.2109375f, 6.04296875f, -4.16015625f, -12.9921875f, 4.3671875f, 0.380126953125f, -9.25f, 7.87109375f, 7.203125f, -8.828125f, 4.97265625f, -4.8671875f, 1.869140625f, -2.490234375f, -12.5859375f, -11.296875f, -1.84765625f, -3.384765625f, -5.921875f, -2.11328125f, 2.734375f, 5.1875f, 2.439453125f, 7.73046875f, 0.41796875f, -0.147705078125f, -4.0546875f, -4.28125f, 10.6328125f, 6.80859375f, 6.64453125f, -0.487060546875f, -0.7392578125f, -3.70703125f, 5.34765625f, -4.87890625f, -6.453125f, 1.9697265625f, -4.01171875f, -0.004924774169921875f, 3.390625f, 12.1328125f, -8.9453125f, -5.08203125f, -0.6806640625f, 0.99267578125f, 6.8359375f, -13.3828125f, 3.12890625f, -15.7734375f, 10.5703125f, -6.09765625f, -2.921875f, -9.109375f, 15.1796875f, -4.4296875f, 8.71875f, 2.599609375f, -9.9921875f, -8.515625f, 1.916015625f, 1.1435546875f, 8.0390625f, -2.1015625f, 4.4140625f, -1.91796875f, 5.9609375f, 4.3359375f, -1.1953125f, -2.4375f, -1.267578125f, -1.4306640625f, 5.515625f, -3.3984375f, 1.08984375f, -9.8203125f, -0.9267578125f, -9.515625f, 10.2734375f, 1.5478515625f, -0.64453125f, -2.681640625f, -3.28125f, 12.9375f, -2.322265625f, 13.9921875f, -4.12890625f, 9.34375f, 7.3203125f, -1.5263671875f, 7.94140625f, 8.3984375f, 1.1708984375f, -11.3828125f, 3.00390625f, 3.185546875f, 10.703125f, 12.9453125f, 4.1875f, -1.9111328125f, 2.197265625f, 1.7255859375f, 5.171875f, -3.947265625f, -8.765625f, -6.1640625f, -6.10546875f, 0.51123046875f, 8.4296875f, -18.359375f, -9.953125f, 9.921875f, 6.3203125f, -0.9892578125f, -9.9765625f, -16.421875f, 22.8125f, 2.978515625f, 2.775390625f, 24.53125f, 13.0234375f, 8.1015625f, -2.046875f, -1.4951171875f, -10.671875f, -16.390625f, -8.78125f, 3.529296875f, -10.390625f, 4.69140625f, 4.41015625f, 0.95654296875f, 0.321044921875f, -1.806640625f, 6.75390625f, 12.015625f, -7.33203125f, 4.30078125f, 9.8828125f, 13.7734375f, -20.265625f, 0.67431640625f, 15.9921875f, 8.9453125f, -10.5078125f, -9.15625f, 0.196044921875f, -2.525390625f, -7.2890625f, -4.015625f, 1.2412109375f, -2.05859375f, 2.263671875f, 5.91796875f, -12.59375f, 0.478271484375f, 8.8125f, 1.8818359375f, -3.298828125f, 6.80078125f, 1.775390625f, 18.859375f, -13.7890625f, 16.828125f, 7.87109375f, 9.234375f, -10.2265625f, -6.66015625f, -6.640625f, -13.0078125f, 5.41015625f, 7.51953125f, -1.4619140625f, -6.8125f, 2.40234375f, 7.10546875f, -13.2109375f, 17.46875f, -6.734375f, -2.953125f, -5.10546875f, -2.05859375f, -5.44140625f, 1.0458984375f, 3.974609375f, -3.89453125f, -2.00390625f, 2.109375f, -6.41015625f, 10.984375f, -0.464599609375f, 0.83447265625f, 2.138671875f, 2.14453125f, -8.34375f, -3.77734375f, 5.4609375f, 0.76220703125f, -4.58203125f, -1.4345703125f, -5.38671875f, 9.6171875f, 1.4755859375f, 7.71875f, -4.94921875f, 0.383544921875f, -8.7265625f, -11.1015625f, -4.19921875f, -5.12109375f, 4.8359375f, -1.419921875f, 2.916015625f, 5.00390625f, -6.703125f, -7.453125f, -0.72021484375f, 1.056640625f, -8.2421875f, -3.01171875f, 11.453125f, 0.5087890625f, -6.07421875f, 3.638671875f, -0.99755859375f, 12.1015625f, 3.310546875f, -2.09765625f, -2.0390625f, 6.75390625f, 5.68359375f, -5.34375f, -10.75f, 3.64453125f, -7.66796875f, -8.375f, 0.416259765625f, -10.984375f, -5.76953125f, -2.044921875f, -2.228515625f, 8.4375f, -7.41796875f, 0.4912109375f, 9.0859375f, 7.16015625f, -1.6552734375f, 7.46875f, -2.916015625f, 10.1328125f, 10.7109375f, 10.1875f, -2.369140625f, 12.625f, -8.203125f, 12.03125f, -1.638671875f, 2.263671875f, 10.7265625f, -12.0234375f, 1.6767578125f, -10.5625f, -2.896484375f, 6.5390625f, -2.513671875f, 16.78125f, -2.232421875f, -0.8896484375f, -7.1328125f, -4.09765625f, -4.109375f, -4.69921875f, 0.98046875f, -7.0f, -2.25390625f, -5.25f, 2.873046875f, -11.484375f, 4.421875f, 4.921875f, -4.4609375f, 1.435546875f, -6.02734375f, -13.578125f, 6.30078125f, -4.7890625f, -1.1650390625f, -3.130859375f, -5.7421875f, 2.392578125f, -3.978515625f, 5.08984375f, 7.58984375f, 0.6416015625f, 4.046875f, 3.228515625f, 3.70703125f, -4.828125f, 10.5234375f, 3.55078125f, -3.3359375f, 11.1875f, 4.16796875f, 1.1640625f, -0.396484375f, 6.48828125f, 5.25390625f, 6.125f, -4.05078125f, -6.64453125f, -5.40234375f, -2.427734375f, 0.2332763671875f, -6.35546875f, 3.78125f, -5.18359375f, -3.7578125f, -9.046875f, -4.625f, 4.078125f, 6.0859375f, -8.59375f, 15.421875f, -5.2734375f, -3.548828125f, 1.5009765625f, 0.5634765625f, 3.865234375f, 4.40234375f, -2.15625f, -0.71044921875f, -9.921875f, 5.87109375f, 14.8359375f, -2.712890625f, 6.26171875f, -7.25f, 0.498046875f, 15.765625f, -0.43212890625f, 8.1796875f, 6.328125f, 5.10546875f, -27.75f, -2.0078125f, -4.3125f, 1.4140625f, 10.8046875f, -3.833984375f, 3.51171875f, 17.375f, 4.078125f, -1.6328125f, -6.45703125f, -2.9375f, 3.548828125f, -4.9921875f, -13.359375f, 5.11328125f, -6.765625f, 3.611328125f, 6.77734375f, 4.09765625f, -4.01953125f, -0.80615234375f, -4.03125f, 1.57421875f, 0.0694580078125f, 5.95703125f, 10.3671875f, -3.927734375f, -1.8603515625f, -7.796875f, 3.5859375f, 8.3828125f, 9.328125f, -1.783203125f, -0.76513671875f, -4.46484375f, -3.708984375f, 3.513671875f, 5.30859375f, 4.62890625f, -0.146240234375f, -1.37890625f, 1.21484375f, 2.54296875f, -16.34375f, 3.15234375f, 7.6328125f, 1.2744140625f, 6.74609375f, -8.7109375f, -10.1015625f, -9.375f, 2.421875f, 13.4296875f, 2.294921875f, 2.529296875f, 2.94921875f, 13.96875f, -3.0859375f, -10.6875f, 9.9375f, -3.603515625f, 8.1796875f, 2.36328125f, 12.3046875f, 7.23828125f, 1.416015625f, 4.109375f, 19.1875f, 12.3203125f, -11.703125f, -1.6064453125f, 3.275390625f, 2.751953125f, -11.5625f, 9.0625f, 0.720703125f, -5.015625f, -1.8935546875f, -1.306640625f, -0.54248046875f, -1.1884765625f, 2.232421875f, 11.1796875f, -9.09375f, 4.31640625f, -4.50390625f, -1.052734375f, -8.7890625f, -12.5078125f, -3.146484375f, 7.31640625f, 6.6171875f, -13.328125f, -0.436767578125f, -9.3125f, -7.9453125f, 0.58740234375f, 6.984375f, -8.578125f, 15.6328125f, -4.625f, 4.08203125f, -12.28125f, -7.77734375f, -2.970703125f, 1.984375f, 11.21875f, -7.28125f, -5.5078125f, 6.921875f, -10.78125f, -16.375f, -3.560546875f, -7.0078125f, 7.890625f, -6.48828125f, 7.4375f, 10.515625f, -6.1796875f, -0.5830078125f, 3.44921875f, 2.86328125f, -2.279296875f, 6.5546875f, 0.701171875f, 11.625f, -5.12890625f, 7.1640625f, -2.841796875f, -7.92578125f, -10.421875f, 8.0859375f, 9.53125f, 3.9375f, 3.279296875f, -5.87109375f, 10.265625f, -8.3671875f, -1.8984375f, -10.4765625f, -4.90234375f, 1.419921875f, 6.21484375f, 14.578125f, 4.2890625f, 8.65625f, -4.48828125f, 3.80078125f, 9.0859375f, 12.375f, -4.68359375f, 10.3515625f, -5.48828125f, 4.828125f, 7.53125f, -10.7265625f, -9.9765625f, -12.90625f, -2.435546875f, 3.59765625f, -0.74853515625f, -2.4921875f, 8.5703125f, -2.134765625f, 5.12109375f, 2.75f, -6.46484375f, 4.5078125f, -3.35546875f, 2.556640625f, -2.078125f, 5.58984375f, 0.2275390625f, 1.556640625f, -3.40625f, -7.01953125f, 0.436279296875f, 2.58203125f, 2.119140625f, 5.171875f, -1.6865234375f, 2.51171875f, 4.66796875f, -16.65625f, 9.6015625f, 0.340087890625f, -2.759765625f, 0.25537109375f, 11.734375f, -7.609375f, 1.759765625f, -6.87890625f, -4.08984375f, 4.40234375f, -9.546875f, 3.818359375f, -7.640625f, 8.6953125f, -4.21484375f, -3.08984375f, -0.281982421875f, -0.685546875f, -6.7265625f, -2.498046875f, 5.2265625f, 9.390625f, 6.5390625f, 4.1171875f, 0.63818359375f, -1.6875f, 5.11328125f, 12.984375f, 9.2109375f, -5.3125f, 1.1357421875f, -4.1953125f, -3.232421875f, -4.47265625f, -11.90625f, 18.359375f, -4.51171875f, -9.1484375f, 4.5859375f, 0.37548828125f, 1.8251953125f, -0.1304931640625f, -9.234375f, -2.337890625f, 1.9228515625f, 1.49609375f, -5.3046875f, 0.07177734375f, 7.44140625f, -0.779296875f, -7.5546875f, 0.828125f, -5.58984375f, 4.37109375f, 0.63232421875f, -2.9140625f, 4.11328125f, -3.64453125f, 2.86328125f, 3.490234375f, -11.09375f, 5.44921875f, 3.4765625f, 20.40625f, -6.35546875f, 1.03125f, -5.15625f, 7.99609375f, -1.140625f, -7.9140625f, 5.046875f, -0.488525390625f, 5.0f, 6.60546875f, 9.5f, -3.3125f, 3.82421875f, 3.654296875f, -9.4765625f, -3.1875f, -3.037109375f, -6.03515625f, 3.23046875f, -2.326171875f, -8.375f, 9.765625f, 6.09375f, -0.1446533203125f, -4.578125f, -8.3515625f, 4.55078125f, -12.484375f, 2.685546875f, -0.31689453125f, -0.63330078125f, 4.92578125f, 18.71875f, 7.47265625f, 3.607421875f, 2.552734375f, -12.71875f, 10.2265625f, 12.6953125f, 12.046875f, -8.890625f, 4.38671875f, -8.09375f, -10.3828125f, 6.14453125f, -9.6640625f, -4.71875f, 8.6328125f, 2.74609375f, 10.6875f, 3.0078125f, -2.615234375f, -2.744140625f, 0.2044677734375f, 5.81640625f, 0.88525390625f, -3.265625f, -11.3046875f, 0.342529296875f, 4.4921875f, -16.53125f, 3.70703125f, 2.732421875f, 1.13671875f, -5.21875f, -8.171875f, -8.09375f, -5.76953125f, -14.890625f, 12.9765625f, 0.201171875f, 5.78125f, -6.29296875f, 0.85302734375f, -1.6650390625f, -3.912109375f, 3.94921875f, 2.0234375f, -8.7421875f, 8.2734375f, 6.48828125f, 2.6953125f, 6.0234375f, 5.81640625f, -1.677734375f, 3.599609375f, -7.83984375f, -3.02734375f, 8.3828125f, 12.5078125f, -1.8984375f, 4.50390625f, 5.3203125f, -4.703125f, 1.5830078125f, -3.775390625f, -10.3359375f, 8.2890625f, -4.97265625f, -13.765625f, -17.453125f, -10.5546875f, -6.86328125f, -7.2578125f, 0.703125f, 13.328125f, 7.7890625f, 9.5f, 0.8935546875f, 8.7109375f, -3.970703125f, 11.3515625f, -9.203125f, -1.4765625f, 2.73046875f, 11.265625f, -1.3447265625f, -7.125f, -5.2265625f, 7.375f, -1.9248046875f, -8.8125f, 7.99609375f, 4.66015625f, -5.4375f, -9.4453125f, -7.0859375f, -2.888671875f, 3.86328125f, -2.388671875f, -1.466796875f, 10.4921875f, 0.890625f, 1.3388671875f, -1.505859375f, -6.92578125f, 0.2366943359375f, 0.82763671875f, -5.77734375f, 16.171875f, 7.73828125f, 7.1171875f, 2.884765625f, -9.421875f, 3.505859375f, -1.2294921875f, -7.0859375f, -0.1376953125f, -10.28125f, 4.234375f, -0.9326171875f, 11.8671875f, -0.69091796875f, -7.39453125f, 11.703125f, -7.98046875f, -2.208984375f, -5.02734375f, 6.03515625f, 2.76171875f, -3.47265625f, 0.36474609375f, -8.234375f, -11.453125f, -11.015625f, 5.44921875f, 12.2265625f, 2.1015625f, 4.84375f, 5.8671875f, 2.47265625f, -0.6484375f, -8.2578125f, 8.234375f, -1.3447265625f, -4.0f, -3.271484375f, 16.515625f, -2.84375f, 6.6796875f, -1.38671875f, 7.80859375f, -1.65625f, 1.61328125f, 3.982421875f, 3.826171875f, -2.533203125f, -0.259765625f, 7.609375f, -4.42578125f, 4.93359375f, -5.53515625f, 5.51171875f, -8.3125f, -5.49609375f, -9.5546875f, 3.38671875f, 11.515625f, 1.697265625f, 6.515625f, -4.50390625f, 5.484375f, -0.40869140625f, 6.5f, 19.921875f, 5.3203125f, -6.984375f, -10.40625f, 3.779296875f, -8.6875f, 7.1640625f, -0.84521484375f, -1.6826171875f, 5.375f, -0.198486328125f, 10.890625f, 10.421875f, 3.859375f, -3.244140625f, 13.5703125f, 11.3671875f, -3.392578125f, -5.09375f, -11.796875f, 1.2685546875f, -8.9375f, 16.0f, -0.96435546875f, -15.296875f, -19.296875f, 0.87939453125f, -4.90625f, 7.85546875f, 5.01953125f, -8.2734375f, -4.23828125f, -2.955078125f, -16.484375f, 0.6123046875f, -12.1328125f, 7.1328125f, -5.8515625f, -5.25f, 2.197265625f, -2.2578125f, -7.79296875f, 7.55078125f, -10.328125f, 6.84375f, -3.685546875f, 0.50244140625f, -21.5f, 2.287109375f, 2.482421875f, -4.21484375f, 19.953125f, 5.69140625f, 1.3203125f, 2.919921875f, -0.4775390625f, -6.453125f, -1.0556640625f, 2.322265625f, 6.5703125f, 5.921875f, -3.97265625f, 12.3203125f, 9.21875f, -1.8427734375f, -2.466796875f, 8.6484375f, 5.1015625f, -2.255859375f, 0.42578125f, -2.384765625f, -8.75f, -4.61328125f, -6.54296875f, -8.1015625f, 4.265625f, -1.3095703125f, -11.640625f, -0.59716796875f, -3.498046875f, 2.494140625f, 12.578125f, 8.0f, -3.75390625f, 1.541015625f, 9.6171875f, 2.34765625f, 3.984375f, -8.765625f, 9.7734375f, 5.90234375f, -9.15625f, 0.56689453125f, -8.1015625f, 15.3046875f, -7.68359375f, 0.30712890625f, 2.357421875f, 3.234375f, -11.0f, 4.58203125f, 4.02734375f, -8.109375f, -1.6962890625f, 1.53125f, 3.85546875f, 2.365234375f, -11.5078125f, 9.046875f, -6.66796875f, 11.6015625f, 0.38916015625f, 7.74609375f, 13.984375f, 3.115234375f, -11.5703125f, -3.421875f, 0.1988525390625f, -6.421875f, 20.234375f, 24.375f, 10.796875f, -7.859375f, 0.358642578125f, 2.173828125f, 0.9599609375f, 15.0390625f, 12.375f, -8.7421875f, -7.37109375f, -1.2490234375f, -6.80859375f, 0.412841796875f, 0.1942138671875f, -1.9423828125f, 4.69921875f, -8.53125f, 6.4453125f}; - -static const __fp16 Apack_dram[4096] - __attribute__((section(".dram"), aligned(64))) = {0.1419677734375, -0.346923828125, 0.8125, -1.98046875, -0.297119140625, 0.05975341796875, 0.6748046875, -1.076171875, -0.72119140625, 1.64453125, -0.62646484375, -1.3271484375, 0.5966796875, -1.5029296875, 0.4951171875, 0.495849609375, -1.6689453125, 0.449951171875, 0.7734375, -0.1544189453125, -1.521484375, -0.2335205078125, 0.98193359375, -0.1842041015625, -1.5, 1.86328125, 1.0087890625, -0.257568359375, 0.027069091796875, -0.36279296875, -0.343994140625, 0.7900390625, -1.33203125, -0.7646484375, -0.52294921875, -0.47998046875, -1.3916015625, -1.09375, -0.7998046875, 0.533203125, 0.218994140625, -0.318359375, 0.80517578125, 1.5166015625, 0.197509765625, 0.10040283203125, 1.00390625, 0.720703125, 0.58251953125, 0.5712890625, 0.135986328125, 0.1827392578125, -0.1173095703125, 1.203125, 0.97802734375, -0.27783203125, 1.9345703125, 2.42578125, -1.318359375, -0.044677734375, 0.7607421875, 1.2294921875, -0.2568359375, -0.2607421875, 0.3740234375, 0.062164306640625, -0.68896484375, -0.486328125, -0.693359375, -0.4228515625, 0.110107421875, 0.484375, 1.4833984375, -0.36328125, -0.16552734375, 0.68408203125, -1.9189453125, -1.3408203125, 0.5771484375, -0.279052734375, 1.755859375, -0.53173828125, 1.4609375, 2.349609375, 0.541015625, -0.39892578125, 0.43212890625, -0.615234375, 0.802734375, -1.134765625, 0.9072265625, 0.9306640625, -2.00390625, -0.65283203125, 1.6171875, 1.171875, 0.1396484375, 1.7412109375, 0.43798828125, 0.544921875, -0.7939453125, -1.6884765625, -0.52001953125, 0.0224609375, -0.244140625, 0.64794921875, 0.198974609375, 1.7685546875, -1.4873046875, 1.15625, -0.45166015625, 0.92724609375, 0.419189453125, -0.48681640625, -0.8525390625, 1.39453125, -0.6162109375, 0.82763671875, -0.36279296875, -1.4765625, -0.2110595703125, -0.50390625, -0.900390625, -1.0087890625, 0.271484375, 0.1343994140625, 0.42822265625, -1.486328125, -0.521484375, -1.33984375, -0.045440673828125, -0.75, 0.53466796875, -1.279296875, -0.1632080078125, 0.405517578125, 0.348876953125, 0.2354736328125, -0.06689453125, -2.705078125, -2.3203125, -1.03515625, -0.98974609375, -0.498291015625, -0.1815185546875, -0.1170654296875, -0.325439453125, -1.3671875, 1.3349609375, -1.619140625, -2.28515625, 1.0908203125, -0.337646484375, 1.22265625, -0.3388671875, -1.3994140625, 0.46728515625, 1.5751953125, -1.0810546875, 0.4697265625, -0.8515625, -0.7158203125, -1.30078125, -1.3623046875, -0.359130859375, -0.60888671875, -0.431396484375, -0.33349609375, -0.1202392578125, -0.7763671875, 0.8369140625, 0.458251953125, -0.465087890625, -0.671875, 0.958984375, 0.41455078125, 2.568359375, 0.56298828125, 1.1767578125, -0.55126953125, -0.033782958984375, 0.302978515625, -0.265380859375, 0.759765625, -0.67822265625, 0.59228515625, 0.80224609375, 0.92236328125, -0.79931640625, 1.546875, 0.6552734375, 1.083984375, 1.0556640625, -0.333984375, -0.29345703125, 1.318359375, -0.333251953125, 2.9375, -0.468994140625, -0.48779296875, -0.2318115234375, -1.0009765625, 1.138671875, 1.1669921875, 0.01279449462890625, -0.69921875, -0.440185546875, 1.544921875, -0.732421875, -0.20654296875, -0.197265625, -1.357421875, 2.85546875, 0.74658203125, -0.216064453125, 1.779296875, -0.7607421875, -0.197509765625, -0.296875, 0.66357421875, -0.1907958984375, -0.09613037109375, 1.5654296875, 1.41015625, -0.78662109375, 1.0244140625, -0.4833984375, -0.408203125, -0.49462890625, -0.39990234375, 2.830078125, -0.85546875, 0.04779052734375, -0.72119140625, 1.21875, 0.919921875, 0.77783203125, -0.5712890625, 0.9404296875, 2.10546875, 1.40234375, 0.43310546875, -1.53125, 0.64208984375, 1.3046875, -0.80859375, 0.0107574462890625, 0.327392578125, -0.42626953125, -0.56298828125, -1.064453125, -1.5556640625, 0.58056640625, 0.12127685546875, -1.57421875, -0.30810546875, -0.34619140625, 1.4052734375, -1.6171875, 0.261962890625, 0.55908203125, 0.9931640625, 0.29296875, -1.34765625, 0.1112060546875, 0.43310546875, 0.6279296875, 1.91015625, 0.814453125, 0.29443359375, 0.322998046875, 1.2958984375, 0.09136962890625, 0.401611328125, -0.04510498046875, -1.1650390625, -2.4140625, -0.7734375, 0.5078125, -1.943359375, 0.1138916015625, 0.472412109375, -0.2406005859375, 1.2177734375, 1.6591796875, -0.2030029296875, 0.54248046875, 1.0810546875, -1.1875, 0.034576416015625, -0.271728515625, -1.23828125, 0.082275390625, -0.443359375, 0.463623046875, 1.873046875, -0.32666015625, 1.6533203125, 1.4052734375, 1.0439453125, -0.8701171875, -1.88671875, -0.90576171875, 0.56005859375, 0.409912109375, -1.224609375, -1.6201171875, 1.4140625, -0.056884765625, -1.3291015625, -0.29150390625, -0.02972412109375, -1.0810546875, -0.63232421875, -1.66796875, -0.2310791015625, -0.2015380859375, -0.6865234375, -0.96728515625, 1.6796875, 0.455322265625, 1.6669921875, -0.1099853515625, -0.85107421875, 0.61669921875, -0.84326171875, 0.0274505615234375, 0.91650390625, 0.054412841796875, -3.046875, 0.6279296875, -0.75146484375, -0.9248046875, 0.759765625, 1.037109375, 0.2021484375, 0.59716796875, 0.1058349609375, 1.6943359375, -0.41064453125, -1.3046875, 0.25439453125, -1.8515625, 0.1500244140625, 0.277587890625, 1.1982421875, 0.265869140625, 0.318115234375, -2.181640625, 0.72314453125, 0.436767578125, -1.490234375, 0.3427734375, 0.54345703125, 0.3505859375, -0.09765625, 2.11328125, 0.419677734375, -2.0390625, -0.330322265625, -0.8955078125, -2.30078125, -1.6875, -0.15234375, 1.7099609375, 1.259765625, 2.24609375, -1.6611328125, 1.26171875, -0.7763671875, 0.374755859375, -0.33251953125, -0.52294921875, -0.67529296875, 0.1317138671875, -0.384521484375, 1.1728515625, -0.69873046875, 1.546875, -0.463623046875, -1.1904296875, 1.3515625, -0.464111328125, 0.658203125, 0.342041015625, 1.2109375, -0.71728515625, -0.6865234375, 0.36376953125, -0.040863037109375, -0.285400390625, -0.4375, 1.23046875, 1.0400390625, -0.488037109375, -0.1829833984375, -0.273193359375, -0.08905029296875, 0.28173828125, -0.7705078125, 0.63037109375, 0.2305908203125, 1.015625, -1.1796875, -0.26513671875, -0.76171875, 0.8955078125, 1.08984375, 0.83740234375, -0.55859375, 0.2156982421875, 0.292236328125, 0.51416015625, 1.0185546875, 0.11572265625, -1.6044921875, 0.779296875, -1.1708984375, 0.83349609375, 0.1658935546875, -1.466796875, -0.56005859375, -1.4111328125, -1.154296875, -0.388427734375, 0.5615234375, -1.6767578125, 2.021484375, 1.572265625, -1.103515625, 0.92822265625, -0.52490234375, -0.97900390625, 0.98388671875, -0.11822509765625, -0.509765625, -1.3369140625, -1.3193359375, 1.5771484375, -0.71728515625, 0.53564453125, 0.473388671875, 3.705078125, -0.2421875, 0.03564453125, -0.386962890625, 0.1419677734375, 1.451171875, 0.4658203125, 0.56640625, 1.11328125, -1.96484375, 1.591796875, -0.962890625, -1.439453125, -0.2208251953125, -0.387451171875, -0.1923828125, 0.8203125, 0.1763916015625, -0.1583251953125, -1.373046875, 0.67578125, 0.2052001953125, -0.94189453125, 0.640625, 0.295166015625, -0.48779296875, 0.34716796875, 0.66796875, 1.3837890625, 0.34326171875, 1.4453125, 0.30908203125, -0.619140625, 1.7060546875, -0.41064453125, 1.4609375, -0.423095703125, -0.297119140625, -0.47314453125, 0.69091796875, -1.306640625, 0.7001953125, -1.1884765625, 0.1451416015625, -0.6123046875, -0.95849609375, -0.91552734375, -0.92578125, 1.087890625, -1.1669921875, -2.36328125, -0.08648681640625, 0.229736328125, -0.63232421875, -0.1827392578125, 0.453857421875, 1.9814453125, 0.39990234375, 0.62255859375, 1.07421875, -0.221923828125, 0.7421875, 1.5712890625, 0.52001953125, -1.4140625, 0.7646484375, 1.1474609375, 0.409912109375, 0.8603515625, 1.728515625, 0.07781982421875, 1.0341796875, -0.50390625, 0.303466796875, -1.369140625, 1.0712890625, -0.6669921875, 2.01953125, 2.248046875, 0.16162109375, -1.6611328125, -1.384765625, 0.0938720703125, -0.033966064453125, -1.228515625, -0.0826416015625, 1.1396484375, 1.2060546875, 0.362548828125, -1.2744140625, -0.67333984375, 0.4130859375, 1.8916015625, -0.34228515625, -0.70703125, 1.3447265625, 1.2431640625, -0.5625, -1.4228515625, 0.1627197265625, 0.7666015625, 0.02752685546875, -1.66796875, -0.239990234375, 0.372802734375, -0.78466796875, -1.017578125, -0.1795654296875, 0.03179931640625, -0.5107421875, -0.7431640625, 1.212890625, 0.109130859375, 0.226318359375, 0.65478515625, 0.1182861328125, 0.294677734375, -0.1353759765625, -0.1558837890625, 0.943359375, -1.80078125, -0.160888671875, 0.57666015625, -0.99267578125, -1.955078125, 0.51416015625, 0.48828125, 0.1917724609375, -0.281494140625, 1.4716796875, -0.1566162109375, 1.0234375, 0.3466796875, 0.0645751953125, 1.2138671875, -0.14501953125, -0.63330078125, -0.78515625, -0.231689453125, 0.2197265625, -0.12274169921875, -0.10491943359375, 0.0670166015625, -0.5517578125, -1.21875, -0.30859375, 0.88525390625, 1.068359375, -0.134521484375, -0.172607421875, 2.240234375, 0.92333984375, 0.335205078125, 2.0390625, 0.03436279296875, 1.0966796875, 0.8154296875, 0.58740234375, -0.10614013671875, -0.150146484375, 1.1953125, 0.420654296875, -0.65087890625, -1.333984375, -1.3349609375, 1.28125, -0.25927734375, 0.0098724365234375, -0.66796875, 0.020660400390625, 0.63525390625, 1.1630859375, 0.52197265625, 2.55859375, 2.798828125, 0.468505859375, -0.04425048828125, -1.595703125, 1.97265625, -1.1611328125, 1.03515625, -2.5078125, 0.399169921875, 0.77978515625, -0.0292205810546875, -2.6171875, 0.599609375, -0.86328125, 0.1314697265625, -0.86279296875, -0.53515625, -0.4150390625, 0.24609375, 1.9228515625, 0.07635498046875, -0.5673828125, 0.53271484375, 0.677734375, -0.47216796875, 0.8681640625, -0.056243896484375, -0.34521484375, 0.0092010498046875, 0.0275421142578125, 0.1456298828125, -0.04974365234375, 0.479736328125, -0.390625, 0.1748046875, 0.301025390625, -0.18701171875, 0.458984375, -0.47314453125, 0.15185546875, -1.5576171875, 1.544921875, 0.1243896484375, -0.99853515625, 2.037109375, -0.95654296875, 0.1455078125, -1.158203125, -0.261962890625, -0.72412109375, -0.326416015625, 1.5048828125, 0.306640625, -0.08892822265625, 0.64453125, -0.09075927734375, -0.5703125, 0.93603515625, -1.1845703125, 0.337890625, -0.7060546875, -0.47021484375, 0.6591796875, -1.7998046875, 1.7744140625, -0.7451171875, -0.29541015625, 0.61962890625, -0.56494140625, -0.0784912109375, -1.8017578125, 0.5224609375, 0.71142578125, -1.0712890625, 1.6689453125, 0.47998046875, -0.15576171875, -1.7578125, -1.05078125, -1.6650390625, -0.96044921875, 0.84814453125, -0.1732177734375, 0.9365234375, 0.351806640625, 1.193359375, 0.466796875, 0.38134765625, 0.10845947265625, -1.6787109375, 0.48828125, 0.365234375, -0.72412109375, 0.458740234375, -0.216796875, -0.63720703125, 1.109375, -0.396240234375, 0.28759765625, 0.14697265625, 0.78515625, 0.006134033203125, -2.78515625, -0.78125, 1.3662109375, -0.3603515625, 0.277587890625, -1.6240234375, 0.323974609375, 0.52587890625, -0.423828125, 0.22119140625, 0.7333984375, 0.72021484375, 0.806640625, 0.35888671875, -1.21875, 0.05419921875, -2.30078125, -0.391845703125, -0.47509765625, -1.3232421875, -1.16015625, 0.333740234375, -0.60888671875, 0.5029296875, -0.7822265625, 1.150390625, -0.546875, -0.2099609375, 0.00824737548828125, 0.97021484375, 1.1328125, -0.392578125, -1.2529296875, -0.1619873046875, -2.62109375, -0.389404296875, -0.2388916015625, -1.240234375, 0.19873046875, -0.178955078125, -0.302001953125, -0.59765625, -0.01409149169921875, 0.59326171875, 0.759765625, 1.58203125, -0.1654052734375, -1.4736328125, -0.728515625, 1.337890625, -1.2783203125, -0.3330078125, 1.3466796875, -0.114013671875, -0.85693359375, 1.1611328125, 0.1458740234375, -0.38720703125, 0.003749847412109375, 1.744140625, -1.498046875, 0.005645751953125, -0.0201263427734375, -1.2421875, -1.4638671875, 0.73828125, 0.480224609375, -0.58740234375, 0.6953125, 1.4013671875, -0.7265625, 1.298828125, 0.308837890625, -0.39794921875, -0.2841796875, 1.2724609375, -0.7666015625, 1.12109375, 0.970703125, 0.63916015625, -0.06353759765625, -0.12939453125, -0.480224609375, -1.3984375, 2.064453125, -0.95263671875, -0.0255889892578125, 0.6982421875, 0.9033203125, -1.126953125, 1.2724609375, 1.181640625, 1.1416015625, 1.1689453125, 0.60205078125, 0.9921875, -0.252197265625, -1.2919921875, -0.3486328125, -0.08660888671875, -0.0836181640625, 0.1468505859375, 0.24853515625, 0.54150390625, 0.447021484375, 2.900390625, 0.6669921875, 1.6630859375, 0.5029296875, 1.0703125, -0.923828125, -0.1973876953125, -1.0751953125, -2.580078125, 0.263916015625, 0.19873046875, 0.38916015625, 0.416015625, -2.033203125, -0.0131378173828125, 0.650390625, 1.28125, -1.2216796875, 1.74609375, 0.470703125, 0.247802734375, -0.83544921875, -0.305908203125, 1.166015625, -0.015655517578125, -0.338623046875, 0.400634765625, -0.8720703125, 0.7109375, 1.58203125, 1.568359375, -2.05078125, 3.30859375, -0.1512451171875, 0.4013671875, -1.4921875, 0.2215576171875, -0.062103271484375, -0.07562255859375, 0.564453125, -1.408203125, 0.51904296875, -0.240966796875, 0.5791015625, 0.52001953125, -1.34765625, -1.3974609375, -0.276123046875, -0.07379150390625, 0.16650390625, 1.4462890625, 0.966796875, 0.7841796875, -0.01480865478515625, -0.87841796875, 0.2032470703125, -0.84326171875, -0.047027587890625, -1.0966796875, 0.447509765625, -0.1309814453125, 0.81982421875, -0.321533203125, 0.29638671875, -1.939453125, 0.8017578125, 0.767578125, -0.8642578125, -0.93212890625, -0.00833892822265625, 0.322021484375, 0.2587890625, -1.23828125, 0.962890625, 1.572265625, -0.33984375, 0.25634765625, 2.80859375, 0.05364990234375, -1.6767578125, 0.0810546875, 0.06744384765625, 0.71337890625, 0.892578125, -1.865234375, 0.313720703125, -0.479248046875, -0.1580810546875, -0.859375, 0.1302490234375, 0.82421875, -0.5263671875, 0.919921875, 0.36572265625, 0.15625, 1.9267578125, -0.231689453125, -0.075927734375, -0.43603515625, 1.060546875, 0.57421875, 0.025177001953125, -1.8251953125, 0.775390625, 0.5791015625, 0.689453125, 1.0205078125, -0.2452392578125, 0.24609375, -0.08770751953125, 0.1796875, -0.56201171875, -1.7998046875, 0.486572265625, 2.125, -0.6103515625, -0.252197265625, -1.9287109375, -0.57958984375, 0.2318115234375, 0.8232421875, -1.5185546875, -2.7578125, -0.4345703125, 0.71826171875, -0.935546875, 1.837890625, -0.01494598388671875, 0.7216796875, -1.994140625, 1.462890625, 1.12109375, 0.15185546875, -1.0185546875, 1.265625, -0.7509765625, 2.7734375, 2.0625, 0.615234375, -1.2900390625, 1.509765625, 1.5400390625, -0.70751953125, 0.327880859375, 0.10601806640625, -1.5166015625, 0.97314453125, -0.182861328125, 1.0869140625, -3.0859375, 0.85205078125, -0.3896484375, 0.32470703125, -1.21484375, -0.2210693359375, -0.63232421875, -0.229736328125, -1.373046875, 0.10931396484375, -0.830078125, 0.556640625, 0.1666259765625, 1.353515625, -1.3857421875, -0.93115234375, -1.5712890625, -1.109375, 0.73193359375, -0.2135009765625, -0.2012939453125, -0.50537109375, -1.1455078125, 1.19921875, -0.52880859375, -0.6591796875, 0.364501953125, -0.90380859375, 1.037109375, 0.385009765625, 0.521484375, -0.0303192138671875, 0.2354736328125, -0.52783203125, -0.84716796875, -0.71923828125, -1.9873046875, 1.203125, 0.440673828125, -0.27001953125, -0.7626953125, -0.1478271484375, -0.85009765625, -0.65185546875, -0.7900390625, -1.1591796875, 0.136474609375, 1.7626953125, -0.8447265625, 0.65869140625, -1.1611328125, -1.048828125, 0.145751953125, 0.28662109375, 1.330078125, 1.166015625, 1.0771484375, -1.21484375, 1.421875, -0.4873046875, -1.9658203125, -0.117431640625, -1.9970703125, 0.41357421875, -0.2266845703125, 0.994140625, -0.85205078125, 0.30517578125, -1.638671875, 1.65234375, 0.64697265625, 0.28857421875, 0.45263671875, -0.55615234375, -0.74169921875, 0.325439453125, 0.062042236328125, -0.7392578125, -0.40625, 0.6611328125, -0.78564453125, -0.61083984375, -1.4453125, 0.2059326171875, 0.9970703125, -0.734375, -0.40380859375, -0.701171875, 2.404296875, -0.1607666015625, 0.1181640625, 1.041015625, 1.1689453125, -0.591796875, 0.51171875, -1.193359375, -1.37109375, -0.65380859375, 1.7001953125, 0.1571044921875, -0.40869140625, -1.830078125, 0.171630859375, 1.3115234375, -2.01953125, -0.49267578125, -0.69921875, 0.427490234375, 0.5068359375, 0.1884765625, -0.9560546875, 0.2200927734375, 0.312744140625, -0.51513671875, 0.233642578125, 1.0732421875, -1.28515625, 0.73388671875, 0.38525390625, 0.426025390625, -0.93408203125, -0.748046875, 0.09088134765625, -1.7626953125, 0.22900390625, -0.474853515625, 1.4482421875, 0.369140625, -1.8154296875, -0.32470703125, 1.0830078125, 1.1455078125, 0.1923828125, -0.0162506103515625, -0.26171875, -0.62353515625, 0.2998046875, -0.1839599609375, -0.122802734375, -1.416015625, 1.048828125, 0.43505859375, -1.8876953125, 0.206298828125, 0.92724609375, 0.17431640625, -2.701171875, -0.075927734375, 1.48828125, -0.07183837890625, -0.474853515625, -0.12237548828125, -1.90234375, -1.501953125, 0.46630859375, -0.07159423828125, -0.415283203125, -1.189453125, 1.056640625, 0.2744140625, -0.66259765625, -0.0609130859375, -1.083984375, -2.32421875, 1.10546875, -0.4775390625, -0.658203125, 1.9013671875, 0.759765625, -0.36376953125, -1.0517578125, -0.80712890625, 0.86962890625, -2.529296875, 0.69677734375, -1.419921875, -1.095703125, -3.12109375, 0.56396484375, 0.89013671875, 1.2333984375, -0.37744140625, 0.00745391845703125, 1.220703125, -0.79443359375, 1.576171875, 0.07086181640625, -0.450927734375, -0.5712890625, -2.388671875, -1.5732421875, -0.1265869140625, -0.66259765625, 0.08526611328125, -0.9931640625, 0.685546875, -0.487060546875, -0.927734375, 0.485595703125, -0.09295654296875, -0.43798828125, 1.8330078125, -1.40625, 0.55859375, 0.83935546875, -0.1602783203125, 0.61572265625, -0.587890625, 1.0693359375, 1.1845703125, 0.10198974609375, 1.7998046875, 0.43359375, 1.3583984375, 0.7451171875, -0.31396484375, -0.1650390625, -0.6767578125, 0.576171875, 2.033203125, -0.10369873046875, -0.02374267578125, 1.025390625, 0.0247039794921875, -0.65966796875, -0.00803375244140625, 1.7275390625, 1.5703125, 0.71484375, 2.099609375, -0.245361328125, -0.403076171875, -0.294921875, -0.759765625, -0.412353515625, 0.05474853515625, 1.23046875, 0.0675048828125, -0.356689453125, 1.5986328125, -0.345947265625, -0.427001953125, 0.4912109375, -1.130859375, -0.4345703125, -0.8076171875, 0.8466796875, -0.826171875, -1.12890625, -2.689453125, 0.806640625, -0.2178955078125, 0.03961181640625, 1.1044921875, -0.8916015625, -0.7529296875, 0.1212158203125, -0.38623046875, -0.07769775390625, 0.1795654296875, -0.349609375, -0.51611328125, 1.4228515625, 0.9375, -1.724609375, 1.2373046875, 1.7568359375, 0.8740234375, -1.7001953125, 0.1947021484375, 0.11541748046875, -0.248291015625, -0.54345703125, -0.46630859375, -0.796875, 1.1591796875, 2.041015625, -1.451171875, 0.8369140625, 0.362060546875, -1.92578125, 0.233154296875, -0.505859375, 0.04351806640625, -0.3720703125, -0.3837890625, 1.048828125, -0.361328125, -0.50146484375, 0.57421875, 1.501953125, -0.54541015625, -2.1796875, -0.284912109375, -1.21484375, -1.5947265625, -0.366455078125, -0.1983642578125, -0.028564453125, -0.56689453125, 0.21044921875, 0.876953125, -2.0546875, -0.2100830078125, -0.2374267578125, 1.4609375, -0.50048828125, 0.63232421875, -0.95361328125, 1.4033203125, 1.421875, 0.583984375, 0.580078125, 0.7421875, -0.0222930908203125, -0.181640625, 0.0054473876953125, 0.623046875, 0.7333984375, -0.9853515625, 0.13720703125, -0.322021484375, -0.01548004150390625, -1.294921875, 0.724609375, -1.2255859375, 1.1142578125, 0.79833984375, 0.1131591796875, 1.3232421875, -0.98388671875, 0.0163116455078125, -1.2060546875, 1.11328125, 0.8017578125, 1.640625, 1.46875, -0.78125, -0.603515625, 1.306640625, -0.374755859375, 1.517578125, -0.72509765625, 0.361083984375, 0.4814453125, -0.045196533203125, -0.1676025390625, -0.8837890625, -0.38232421875, 0.260498046875, -0.810546875, 1.015625, -0.376220703125, 0.55615234375, 0.7451171875, 0.357177734375, -0.334716796875, 0.2685546875, 1.337890625, -0.08856201171875, 0.260009765625, 0.26416015625, -0.47607421875, 0.1917724609375, -1.052734375, -0.5908203125, 2.650390625, 0.76953125, -2.28515625, -0.78759765625, 0.8818359375, -0.1015625, 0.004024505615234375, 0.093994140625, -0.455322265625, -0.09454345703125, -0.9072265625, -1.208984375, -1.50390625, -0.0119781494140625, -0.7392578125, 0.88427734375, 1.193359375, -1.060546875, -0.5126953125, -0.40234375, 1.9228515625, 0.9208984375, 0.7939453125, -0.78369140625, -2.71875, -0.7001953125, 2.3828125, -0.2459716796875, 1.6845703125, 1.4326171875, 2.138671875, -0.46875, -0.5986328125, -0.650390625, -0.7197265625, 0.8642578125, 0.247802734375, -1.3271484375, -0.318115234375, 0.2261962890625, 1.2744140625, 1.31640625, 0.646484375, 0.431884765625, 0.1488037109375, 0.271484375, -1.220703125, 0.59423828125, 0.070556640625, 0.64404296875, 0.9619140625, 0.50634765625, 0.07073974609375, 1.3779296875, 1.298828125, 1.662109375, 1.0439453125, 0.1029052734375, 1.1796875, -0.66552734375, -1.49609375, 2.6328125, 0.57373046875, 1.0693359375, -1.2763671875, -0.400146484375, -0.341796875, -0.154052734375, 0.9775390625, -0.6396484375, -1.2578125, 0.6572265625, 2.0078125, -0.4365234375, 1.1064453125, 0.25439453125, -1.5693359375, 0.0343017578125, 1.375, 1.0830078125, -0.814453125, -2.693359375, -1.3427734375, 0.407470703125, -1.056640625, 1.0634765625, 1.005859375, 0.8251953125, -1.62109375, 0.1876220703125, 0.3203125, 0.2958984375, -0.62353515625, -0.281005859375, 2.08984375, 0.67724609375, -0.56982421875, 0.30712890625, -0.6943359375, -0.2158203125, 0.41015625, -1.482421875, -0.640625, -0.53369140625, -1.19140625, 0.54443359375, -0.9921875, 0.83154296875, 0.2607421875, 0.1646728515625, -0.35009765625, -1.7412109375, -1.2587890625, -1.1064453125, 1.3623046875, 0.1324462890625, -0.281005859375, -0.192138671875, -1.4970703125, 0.474365234375, -0.466552734375, 0.91455078125, 0.428466796875, 2.498046875, 1.310546875, -1.6513671875, 0.263671875, 0.8505859375, -2.001953125, -0.44091796875, -0.389404296875, -0.88330078125, 0.260986328125, 0.465576171875, -0.2135009765625, 1.2392578125, 1.951171875, -1.5380859375, -0.80224609375, 0.0022220611572265625, -0.8525390625, 0.98583984375, 1.6572265625, -1.9111328125, -0.51611328125, -0.044647216796875, 0.14208984375, 0.1217041015625, -1.4541015625, 0.6376953125, 0.54345703125, 0.214111328125, -0.71533203125, 1.5771484375, 0.11553955078125, 0.8916015625, -0.71044921875, 0.51025390625, 0.26611328125, 1.5673828125, -0.205322265625, -0.4345703125, -0.96240234375, 0.451416015625, 0.004810333251953125, -0.90087890625, -1.6416015625, -0.491943359375, -1.431640625, 1.544921875, 0.12255859375, 0.051788330078125, -0.46337890625, 0.2220458984375, -0.7802734375, -1.302734375, 0.127197265625, -0.280517578125, -0.024871826171875, 2.197265625, -0.79638671875, 0.1512451171875, 1.6416015625, 1.056640625, -0.52880859375, -0.290283203125, 0.91796875, 1.2978515625, 0.448486328125, 0.1905517578125, 0.346923828125, -0.525390625, 0.007293701171875, 0.274658203125, 1.939453125, 3.0, -2.3359375, -0.454833984375, -0.5302734375, -0.5576171875, 2.4765625, -0.0070953369140625, 0.10589599609375, 0.289306640625, -0.89794921875, -0.8935546875, -0.113525390625, -1.4365234375, 0.427978515625, 0.0987548828125, 0.104736328125, 0.147216796875, -0.29052734375, 0.99951171875, -0.249755859375, 1.71875, 0.05804443359375, -1.2470703125, 0.5166015625, -2.89453125, -0.1556396484375, -0.326416015625, 1.0517578125, -1.029296875, 0.83447265625, -0.039031982421875, -0.2156982421875, 0.283203125, -0.353759765625, 0.498046875, 0.66552734375, 1.1025390625, 0.2061767578125, -1.0361328125, -1.6689453125, -0.36083984375, 0.0009708404541015625, -0.82275390625, -0.0628662109375, 1.8798828125, -2.03125, -0.01015472412109375, -0.705078125, 0.50439453125, -1.3125, 0.369384765625, 1.1611328125, -0.84033203125, 1.1787109375, -1.0859375, 0.7978515625, 1.05078125, 0.2269287109375, -1.607421875, -1.345703125, -0.135009765625, -0.8271484375, 0.498291015625, 1.9521484375, 0.603515625, 0.053619384765625, -0.966796875, 1.466796875, -1.013671875, -1.1748046875, 0.68994140625, 0.07135009765625, 1.0224609375, 0.2479248046875, 0.52392578125, -1.3017578125, -0.58203125, 0.03009033203125, 0.096435546875, 0.8720703125, -1.6689453125, 0.5517578125, 0.76806640625, -0.93798828125, -0.478759765625, -1.828125, -1.5107421875, -0.171630859375, 1.2236328125, -0.475830078125, 1.2900390625, 0.062347412109375, 1.720703125, 0.480712890625, 0.322021484375, 0.2198486328125, -0.2166748046875, -0.505859375, -0.453369140625, 0.306884765625, 1.3291015625, -0.408203125, 0.235595703125, -0.6005859375, 0.66455078125, -1.2373046875, -0.006229400634765625, -0.362060546875, -0.147216796875, -0.95849609375, -1.087890625, -0.310791015625, -0.56689453125, 0.8564453125, -0.0635986328125, 1.205078125, -0.329833984375, -1.2802734375, 0.097412109375, -0.3212890625, -0.947265625, -0.19921875, 0.31591796875, 0.822265625, -1.9091796875, 2.048828125, -1.3212890625, -1.791015625, 0.56298828125, -0.1102294921875, -0.79736328125, -0.6669921875, -0.544921875, -1.6005859375, -0.63623046875, -0.7998046875, 1.2412109375, -0.0015869140625, -1.720703125, -0.533203125, -0.0280303955078125, -0.77490234375, 0.78466796875, -1.1552734375, 3.419921875, -0.5615234375, -0.3974609375, -2.654296875, -0.1546630859375, 0.97216796875, 0.56201171875, 0.264892578125, -1.1337890625, -0.358154296875, 1.005859375, -0.169189453125, -0.055206298828125, 1.87109375, -1.9541015625, -0.1419677734375, 0.0270843505859375, 0.1224365234375, 0.52099609375, -0.42724609375, -1.4111328125, -0.89892578125, 0.151611328125, 0.1861572265625, 0.055023193359375, -0.7734375, 1.109375, 1.7099609375, -0.387939453125, -1.4501953125, 0.248779296875, 0.2313232421875, 0.345947265625, -1.435546875, -1.087890625, 0.74658203125, -0.65966796875, 0.88525390625, -0.708984375, -0.497314453125, -1.5263671875, -0.36474609375, 0.78271484375, -0.423095703125, 0.56005859375, -0.86376953125, -0.14794921875, -0.4296875, -0.1595458984375, -0.77880859375, 0.68603515625, 0.065185546875, -0.431396484375, -1.8369140625, -0.1190185546875, -0.1693115234375, 2.21875, 1.2119140625, 0.2841796875, 0.005802154541015625, 0.52197265625, -2.087890625, 0.236328125, -0.2479248046875, 0.09930419921875, 0.77294921875, 0.5654296875, 2.576171875, 0.296875, -0.1495361328125, -1.455078125, -0.130126953125, -0.38671875, -0.06878662109375, -1.81640625, -0.5556640625, -0.81787109375, 0.52490234375, -0.08355712890625, -0.90869140625, -1.91015625, -0.767578125, -0.85595703125, -1.103515625, -1.1591796875, -0.8505859375, 0.395751953125, 0.33447265625, 1.6982421875, 0.62841796875, -1.2021484375, 0.0426025390625, -0.2078857421875, 0.939453125, 0.307373046875, -1.0654296875, 0.0025730133056640625, 0.251953125, 0.119873046875, 0.02264404296875, -1.7333984375, 0.1253662109375, -0.00743865966796875, -0.5517578125, -0.468017578125, 0.6240234375, 0.09771728515625, 1.052734375, -0.7841796875, 0.4140625, -0.27294921875, -0.2020263671875, 0.465576171875, -1.716796875, -0.1973876953125, 0.619140625, -1.1689453125, -0.458251953125, 2.1875, 1.25390625, -0.438232421875, 0.72509765625, -0.3408203125, 1.5859375, -0.1685791015625, 0.6044921875, 1.041015625, 0.83740234375, 0.51953125, -1.169921875, 0.383544921875, 0.99462890625, -0.54736328125, 1.7197265625, -0.91015625, 0.04986572265625, 0.272216796875, -0.79052734375, -0.2122802734375, 0.76318359375, 2.05078125, 1.2724609375, -0.61669921875, 0.6435546875, -0.525390625, -0.4599609375, -0.27783203125, -0.59765625, -0.13720703125, 1.884765625, -0.3896484375, -0.4189453125, -1.2568359375, 0.99365234375, 0.50439453125, 2.287109375, -2.482421875, -0.83642578125, 1.216796875, 0.057037353515625, 0.11962890625, -0.53271484375, 0.640625, -1.6181640625, -0.00823211669921875, 1.8388671875, 2.3359375, -0.293212890625, 0.58740234375, -0.65478515625, -0.88916015625, 0.290771484375, -1.6513671875, 0.7509765625, 0.81787109375, 0.8095703125, 0.74267578125, 1.71875, 1.166015625, -0.06591796875, -0.6064453125, -1.6328125, 1.0556640625, 2.23828125, 0.11834716796875, 0.96533203125, 0.67333984375, 0.0838623046875, -0.8349609375, 0.5966796875, 0.75244140625, -0.11981201171875, 1.3798828125, 1.154296875, -0.2015380859375, -0.64453125, 0.689453125, -0.349853515625, -0.07183837890625, -0.444580078125, -0.87744140625, -0.615234375, 2.3984375, 0.79833984375, 0.08477783203125, -0.99609375, 0.11376953125, 0.50634765625, -0.2039794921875, 0.9169921875, 1.236328125, -0.28515625, 1.2841796875, -0.57763671875, -1.921875, 1.4306640625, -1.056640625, -0.52490234375, 0.8291015625, 1.984375, 1.740234375, -0.8564453125, 0.99560546875, 1.2724609375, -0.7373046875, -0.44677734375, 1.828125, -0.42626953125, -0.71337890625, -0.0078582763671875, 0.9677734375, -0.72900390625, -0.2108154296875, 0.445556640625, 1.8505859375, 0.09356689453125, 0.65673828125, 0.0121307373046875, -0.79443359375, -0.63330078125, -1.208984375, -1.044921875, -0.97314453125, -1.240234375, 0.202392578125, -0.368896484375, -1.7451171875, -0.401123046875, -1.849609375, -0.681640625, -2.4921875, 1.2861328125, 0.225341796875, -0.38037109375, 0.478759765625, 0.19140625, 0.445556640625, -1.1689453125, 0.73095703125, 1.19140625, 0.01082611083984375, -1.1640625, -2.09375, -0.88427734375, 2.404296875, -1.4501953125, 0.10113525390625, -0.290771484375, -0.591796875, -0.431396484375, 1.1279296875, -1.013671875, 0.1923828125, 0.95361328125, -0.051605224609375, -0.061370849609375, 0.33154296875, -0.149658203125, 2.23046875, -2.228515625, 0.869140625, -0.2413330078125, 0.87744140625, 0.47314453125, -2.1484375, -1.0732421875, -1.1201171875, 0.82763671875, -2.70703125, 0.04132080078125, -1.7890625, -1.1474609375, 0.439208984375, -0.98388671875, -0.611328125, -0.8642578125, -0.495361328125, -1.630859375, -1.337890625, -0.1837158203125, -0.298828125, -0.333251953125, -0.1181640625, 0.405517578125, -0.85400390625, 1.0625, 0.83251953125, -0.16015625, -0.1318359375, -1.2685546875, -0.2939453125, -1.076171875, -0.2110595703125, -1.2421875, -2.623046875, -0.6083984375, -0.5615234375, -0.79638671875, 1.158203125, -1.0068359375, 0.099853515625, 0.4736328125, 1.9931640625, 0.36474609375, -0.2083740234375, -0.54638671875, -0.4521484375, -1.4560546875, 0.463623046875, -1.708984375, 0.2320556640625, -0.1484375, 1.64453125, 0.107177734375, -0.35400390625, 0.9013671875, -1.9365234375, 0.9375, 0.314453125, 0.4326171875, -0.01363372802734375, 0.61669921875, -1.2177734375, -1.388671875, -0.06024169921875, -0.3876953125, 0.66357421875, 0.4287109375, -0.1583251953125, 0.12432861328125, 2.689453125, 1.1474609375, -3.1953125, -1.0849609375, -0.361083984375, -0.1136474609375, -0.94580078125, 0.80419921875, -0.869140625, 0.94873046875, -3.115234375, 0.58984375, 0.890625, -1.33203125, 1.041015625, 0.59521484375, 0.56689453125, -1.046875, 1.7880859375, 0.74609375, 1.189453125, -1.7578125, -0.71875, 1.0078125, 0.57421875, -1.3583984375, -0.1573486328125, -0.27001953125, 1.09765625, -0.357177734375, -0.119140625, 1.7958984375, -1.8046875, -0.1251220703125, 0.60498046875, -1.0498046875, -0.44091796875, 0.7353515625, 1.42578125, 0.27685546875, -0.369873046875, 0.69775390625, 1.6689453125, 0.93212890625, -0.75634765625, 0.09912109375, -1.427734375, -0.9130859375, -1.279296875, -0.9052734375, -0.83544921875, 0.345703125, -1.138671875, -0.806640625, 1.32421875, 0.2724609375, 1.0751953125, 0.20166015625, 0.137939453125, -1.3798828125, -0.04351806640625, 1.552734375, -1.8466796875, -2.55859375, -0.4853515625, -0.10418701171875, 0.9228515625, -1.099609375, -0.186767578125, -0.73486328125, 0.450439453125, -1.365234375, 0.180419921875, 0.5224609375, 0.31494140625, -0.318359375, 0.66748046875, -0.9091796875, 4.01171875, 0.460205078125, 0.13037109375, -0.153564453125, -0.85400390625, -1.6748046875, 0.276123046875, -2.720703125, -0.404541015625, 0.123046875, -0.1651611328125, 0.1256103515625, 1.3515625, -0.275390625, 1.1181640625, 1.921875, 2.806640625, -0.6640625, -2.181640625, -0.31298828125, 1.24609375, 0.77734375, 0.5078125, -1.1318359375, 0.859375, 0.78125, 1.3828125, -0.31591796875, -0.414306640625, -1.3935546875, 0.88818359375, 0.61865234375, 1.173828125, -1.0859375, 0.340087890625, 0.188232421875, 1.1650390625, 0.2252197265625, 0.7080078125, 0.25732421875, 0.49609375, -1.6708984375, 1.8759765625, 0.2122802734375, 0.394287109375, -0.64453125, -1.314453125, -1.19921875, -2.705078125, 0.1275634765625, 1.3798828125, 1.4609375, 0.69775390625, 0.7392578125, 0.1461181640625, -0.2462158203125, -0.427978515625, 0.09356689453125, 1.3505859375, 0.41064453125, 0.7509765625, -0.052734375, 0.56689453125, -0.0555419921875, -0.67138671875, -0.39599609375, -1.0625, 0.311279296875, -0.1951904296875, -0.074462890625, -0.301513671875, -1.185546875, -1.501953125, 0.1824951171875, 0.1436767578125, 0.2073974609375, -1.611328125, -1.7919921875, 1.015625, 3.419921875, -0.1302490234375, 0.55712890625, -1.41015625, 1.4697265625, -1.6376953125, -0.343505859375, -0.06622314453125, -0.352294921875, -1.7333984375, 0.6669921875, -0.51611328125, 0.077392578125, -1.23828125, -2.322265625, -0.68603515625, -0.77099609375, -0.287109375, -0.04669189453125, -0.98291015625, 0.57080078125, 0.525390625, 0.52294921875, -0.3447265625, -2.361328125, 0.25390625, -1.8984375, 0.350830078125, 1.0927734375, -0.6396484375, -0.1009521484375, 0.8349609375, 0.66259765625, -0.160400390625, -0.38916015625, 0.13134765625, -0.5888671875, 0.65625, -0.261962890625, 1.2548828125, 0.1961669921875, 1.4443359375, -0.296875, 0.400390625, 0.059844970703125, 1.0107421875, -1.2080078125, 1.0693359375, 1.212890625, 0.408447265625, 0.61328125, 0.91845703125, -0.7333984375, 0.0537109375, 1.716796875, -0.7333984375, 0.6484375, 0.3046875, 1.1396484375, -0.5439453125, -0.76171875, -0.064697265625, 0.182861328125, -0.377197265625, -0.08551025390625, 1.98046875, -0.402099609375, 1.431640625, 0.30322265625, -1.3720703125, -0.27197265625, -1.6337890625, -0.244873046875, 0.244873046875, -0.70703125, -0.513671875, 0.72900390625, -0.451416015625, -0.09979248046875, 1.0361328125, 0.03753662109375, -1.671875, -1.458984375, 0.456787109375, -0.525390625, -0.88623046875, -2.376953125, 0.27294921875, 0.2401123046875, -1.3232421875, 0.1536865234375, 0.0556640625, 0.34716796875, -0.000774383544921875, 0.1898193359375, -3.13671875, 0.422607421875, -0.2127685546875, 1.650390625, 1.759765625, 0.99169921875, 0.420166015625, -0.66015625, -2.755859375, 1.447265625, -0.281494140625, -1.662109375, 0.54345703125, -2.4296875, -0.01861572265625, -1.0810546875, -0.84033203125, 0.95654296875, 1.5595703125, -1.0361328125, 0.1976318359375, 2.04296875, 0.3720703125, 0.43896484375, -0.94677734375, 1.0224609375, 1.1171875, -1.4853515625, 0.046112060546875, 1.4375, -0.293212890625, -0.6650390625, -0.94970703125, -0.401123046875, 0.10113525390625, 0.66796875, 1.875, -1.7646484375, -1.02734375, 1.3330078125, 0.68310546875, -0.90576171875, 0.1956787109375, 0.9150390625, 0.1697998046875, 0.21240234375, -1.2119140625, -0.4990234375, -0.60107421875, 1.13671875, 0.25634765625, 0.06817626953125, 1.46875, -1.119140625, -1.072265625, 0.2386474609375, -0.223876953125, 0.489990234375, 0.57275390625, -0.12127685546875, -0.00036334991455078125, -1.0615234375, 1.548828125, -0.2222900390625, -1.900390625, -0.49658203125, 0.927734375, -0.23046875, -1.3017578125, 0.9208984375, 0.435791015625, -0.22412109375, 0.1331787109375, 0.193115234375, 1.619140625, -0.111083984375, -0.1326904296875, 0.70849609375, 1.6591796875, 0.322265625, -0.4501953125, -0.615234375, -0.458251953125, 1.484375, 0.41015625, 1.333984375, -0.1513671875, -1.158203125, -0.25537109375, 1.7578125, 0.82861328125, -0.8203125, 0.9970703125, -0.31591796875, -1.279296875, 0.953125, 0.4306640625, 0.22265625, 1.3486328125, 1.2060546875, 1.6201171875, -1.9306640625, 1.6337890625, -1.33203125, 0.52685546875, -0.325927734375, 0.401611328125, -0.72314453125, -1.0185546875, 0.365234375, 1.26953125, -1.251953125, 0.7392578125, -0.444091796875, 1.62109375, 0.38427734375, 0.8408203125, 0.222900390625, -1.2958984375, -0.237060546875, -0.16748046875, -1.8828125, 0.57861328125, -1.21484375, 0.7255859375, 1.541015625, -0.50439453125, 0.6220703125, 2.705078125, 0.62451171875, -0.06988525390625, -0.25439453125, 1.4970703125, 0.5107421875, 0.034942626953125, 0.82958984375, 0.01549530029296875, 0.66748046875, -0.340576171875, 0.338134765625, -0.0653076171875, 1.744140625, -0.9697265625, 0.239013671875, -0.626953125, -0.0086669921875, -0.053863525390625, 0.5966796875, -2.537109375, -0.1319580078125, -0.73291015625, -0.3603515625, 0.65576171875, -1.4423828125, 0.298583984375, -1.150390625, 0.055145263671875, 2.337890625, 1.873046875, 0.9638671875, -0.440185546875, 0.485107421875, 2.16015625, -0.2724609375, -0.8955078125, 0.128173828125, -0.87060546875, 0.53759765625, 1.4814453125, 0.92822265625, 0.93994140625, 0.4794921875, -0.4189453125, -2.3828125, 0.191650390625, 1.3642578125, 1.0673828125, 0.6796875, 1.4521484375, -0.10137939453125, -0.2225341796875, 1.302734375, 1.482421875, -0.270263671875, -0.1136474609375, 0.658203125, -0.1990966796875, -0.98876953125, 1.3212890625, -1.013671875, 0.57568359375, -0.375244140625, 0.32568359375, -1.9228515625, -1.060546875, 0.359130859375, 0.352294921875, 0.5947265625, -0.2086181640625, 0.7861328125, -1.80078125, 0.1787109375, -0.287353515625, 0.6484375, 2.26171875, -1.1962890625, 1.0341796875, 0.11517333984375, 0.71728515625, 0.802734375, -0.81005859375, -2.673828125, -1.5654296875, 0.80615234375, 0.74951171875, -0.390380859375, 0.38232421875, -0.372802734375, 0.5546875, 0.80859375, 1.765625, 0.08917236328125, 0.50732421875, 2.330078125, -0.583984375, -0.6796875, 0.99072265625, -1.40625, 0.260498046875, 0.68701171875, -0.483642578125, 0.345947265625, -1.70703125, 0.54052734375, 0.374267578125, -0.0008740425109863281, -0.097900390625, -0.9560546875, -0.4453125, -0.047760009765625, 0.50927734375, -0.5419921875, -1.6640625, -1.20703125, -0.54931640625, 1.06640625, -0.056488037109375, -0.89990234375, -0.1339111328125, 0.118896484375, -0.56787109375, -0.283935546875, 0.264892578125, 0.306396484375, -1.052734375, 0.5234375, -0.410888671875, 2.01171875, -1.435546875, 0.342041015625, -0.158203125, 1.73828125, 0.70703125, 2.845703125, 1.078125, -0.4619140625, 0.50341796875, -1.087890625, -0.0367431640625, -0.69140625, 0.748046875, 1.2373046875, 1.25390625, 0.2242431640625, -0.373291015625, 0.42626953125, 0.1390380859375, 0.89892578125, 0.91357421875, 1.896484375, 1.1611328125, 1.0537109375, 0.3720703125, 1.4580078125, 0.252197265625, -0.62548828125, -0.04168701171875, 0.0767822265625, 0.98095703125, 1.3466796875, -1.013671875, -0.1336669921875, 0.8994140625, -0.6494140625, -0.408203125, 0.1981201171875, -0.09588623046875, 2.255859375, -1.2001953125, 0.2197265625, -0.1541748046875, -0.77099609375, 0.91162109375, -0.477783203125, 0.472900390625, 0.51416015625, -0.65966796875, 0.01336669921875, 0.1983642578125, -0.425537109375, 0.92529296875, 0.765625, -0.492919921875, 0.53369140625, -1.0771484375, -0.1712646484375, -2.037109375, -0.54541015625, 0.2900390625, -0.45849609375, -0.09454345703125, 0.060760498046875, -0.1822509765625, 1.755859375, -0.12457275390625, 1.4599609375, -1.1630859375, -1.6865234375, 0.48681640625, 0.91845703125, -0.10601806640625, 0.410888671875, -0.410400390625, -0.4326171875, 1.3544921875, 1.3544921875, 1.146484375, 0.45068359375, -0.83740234375, 1.775390625, -1.26953125, -0.0799560546875, 1.7861328125, 0.1827392578125, 0.5419921875, -0.455810546875, -0.3671875, 0.78125, -1.0869140625, 0.76025390625, 0.332275390625, 0.07861328125, 1.22265625, -1.0361328125, -1.30078125, 0.111083984375, -0.203857421875, 0.54638671875, 3.115234375, 0.765625, -1.3759765625, 1.7900390625, -0.0058746337890625, 0.1416015625, -0.467041015625, -0.0119781494140625, -1.009765625, 1.44921875, -0.212646484375, 0.347900390625, -0.9208984375, -0.97314453125, 0.7998046875, 0.357177734375, 0.494384765625, 0.73828125, -0.91357421875, 1.421875, -0.060943603515625, 0.97216796875, 0.7548828125, -0.9765625, -0.2646484375, 1.1533203125, -0.44873046875, -0.86083984375, 0.0972900390625, -1.2294921875, 1.724609375, 0.1920166015625, -0.50341796875, -1.3076171875, -0.048858642578125, 0.0213470458984375, -0.23974609375, 0.72802734375, -0.5634765625, -1.576171875, -0.6396484375, -2.333984375, 0.30126953125, -0.88330078125, 0.07183837890625, -0.107666015625, -1.1669921875, 1.5263671875, 1.0361328125, 1.001953125, 1.4765625, -1.419921875, 0.5439453125, 0.52392578125, -1.00390625, -0.75341796875, -0.9384765625, -1.4013671875, 0.63720703125, -0.327880859375, 0.810546875, -1.0576171875, -0.5087890625, -0.5439453125, 0.5224609375, 1.46875, -0.79833984375, -0.470458984375, -1.2958984375, 0.486572265625, 0.0809326171875, 1.501953125, 0.05621337890625, 0.62255859375, 1.046875, -0.97900390625, 1.1953125, -0.66259765625, -0.2919921875, 0.149169921875, 2.33984375, -0.09320068359375, 1.03125, -1.1337890625, 0.27197265625, -0.345458984375, 0.18701171875, 1.6259765625, 0.35986328125, -1.0849609375, 0.83642578125, 0.189697265625, 1.5810546875, -0.93798828125, 1.763671875, -0.8271484375, 1.4404296875, 0.37109375, -0.87353515625, -1.2666015625, -0.86181640625, 0.274169921875, -0.447998046875, 0.97705078125, -1.1904296875, 1.3583984375, -0.64599609375, -1.123046875, 0.305419921875, -1.0107421875, 1.3740234375, -0.414794921875, -0.68212890625, -0.256103515625, 1.05859375, 1.076171875, -0.170166015625, 0.439697265625, -0.70068359375, -1.501953125, -0.252685546875, -0.390380859375, 0.958984375, 0.28125, -2.0703125, 0.99072265625, 0.45849609375, 1.3447265625, 0.689453125, -0.327880859375, -0.5830078125, -1.09375, 0.260986328125, 0.305419921875, 0.77880859375, -0.89892578125, -0.83642578125, -0.6650390625, 0.401123046875, -3.0390625, 0.2744140625, -0.53466796875, -0.167236328125, 0.279541015625, -0.6416015625, 0.63330078125, 1.978515625, -0.36669921875, -0.7080078125, 0.0574951171875, 2.134765625, 0.0667724609375, 0.409912109375, 0.1990966796875, -0.6962890625, -0.1705322265625, -0.473388671875, 0.662109375, -0.02056884765625, 0.41455078125, 0.517578125, -0.476318359375, -0.6455078125, -0.8447265625, 1.4296875, 0.5068359375, 0.60791015625, -0.487548828125, -0.13330078125, -0.296630859375, 0.9296875, 1.1806640625, 0.14013671875, 1.794921875, 0.56005859375, 0.8974609375, 0.5185546875, 0.1441650390625, 1.302734375, 0.291748046875, -0.728515625, -1.1162109375, -0.21484375, 0.08709716796875, 0.424072265625, -0.86328125, -0.84765625, -0.61962890625, 0.8388671875, 0.0863037109375, -0.1282958984375, 0.41845703125, 1.9990234375, -1.095703125, 0.56298828125, -1.5869140625, -0.250732421875, -1.0126953125, 0.65380859375, -1.31640625, 0.53564453125, 2.09375, 0.56591796875, 1.62109375, 0.13671875, -0.397705078125, -2.490234375, 0.6484375, 1.5263671875, 0.50048828125, -0.7138671875, 0.68994140625, -0.219482421875, 0.7900390625, 1.4375, 0.436279296875, 0.2059326171875, 1.8046875, -1.03515625, -1.9677734375, -1.2275390625, 2.673828125, -0.1676025390625, -2.150390625, -0.984375, -1.3876953125, 1.4033203125, 1.0693359375, 0.483154296875, 0.7568359375, -0.53515625, -0.7978515625, -0.10272216796875, -0.23486328125, 0.42919921875, -0.33056640625, -0.342529296875, 0.51318359375, 0.6533203125, 1.9169921875, 2.7109375, 0.88330078125, 0.07940673828125, -0.54248046875, -0.7255859375, -1.23046875, -0.64208984375, -0.72021484375, -0.1329345703125, 1.4609375, -0.71630859375, -1.19140625, -0.2578125, 0.404052734375, 0.11590576171875, -0.257080078125, -0.6220703125, -0.09417724609375, -0.95751953125, -0.92041015625, -1.587890625, -1.40625, 0.96142578125, -0.88818359375, -0.48046875, -0.3955078125, 1.1748046875, -0.7607421875, 0.20458984375, -0.58349609375, -0.55517578125, -0.392822265625, 0.313232421875, 0.04986572265625, -0.0162353515625, 2.341796875, -0.4375, 0.8935546875, 1.1982421875, 0.23681640625, -0.37646484375, -0.92919921875, 1.5302734375, 2.04296875, 0.77587890625, 0.1884765625, 0.68798828125, -1.8720703125, 0.6875, -0.5498046875, -1.515625, 0.49560546875, 0.92578125, -1.263671875, 1.7109375, -0.82421875, -0.2158203125, 0.7958984375, -1.2119140625, -0.358642578125, 2.12890625, 0.67138671875, 0.78125, -1.408203125, 0.751953125, 0.478759765625, 0.357421875, 0.53564453125, 0.1766357421875, 0.9599609375, 0.84326171875, -0.72216796875, -0.232421875, 0.51171875, 0.1514892578125, -1.212890625, 0.2237548828125, -0.373779296875, 1.59375, 0.4990234375, 1.2509765625, 0.301025390625, 0.302978515625, -0.6201171875, 0.091064453125, 3.80078125, -0.76025390625, -0.3466796875, -0.2763671875, 0.1533203125, -0.349365234375, 0.47412109375, 0.9306640625, -0.31201171875, -1.501953125, -0.83154296875, 1.017578125, 0.74609375, -0.94482421875, 0.783203125, 0.65185546875, -0.418212890625, 0.61865234375, -0.625, 0.495849609375, 0.65673828125, 0.1766357421875, -0.369140625, 0.1961669921875, -1.3056640625, -1.5, 0.202392578125, 0.322509765625, -0.6796875, 1.0556640625, -1.576171875, 1.1767578125, 0.353271484375, -1.09375, -0.27099609375, -0.48583984375, 0.8125, -0.87451171875, -0.325927734375, -1.654296875, 0.35205078125, 0.053955078125, -0.9814453125, -0.86474609375, -2.13671875, 0.8447265625, -0.9208984375, 1.1943359375, -2.197265625, 0.489990234375, -1.1640625, -0.094482421875, 0.2457275390625, 3.626953125, 0.490966796875, 0.97021484375, -0.1422119140625, 0.16259765625, 1.716796875, -0.0657958984375, 0.1490478515625, 0.256103515625, 0.82177734375, -1.111328125, -0.47216796875, -0.50927734375, -0.5419921875, 1.39453125, -0.39501953125, 1.6865234375, -1.697265625, 1.576171875, 0.89453125, -0.464111328125, 0.8046875, -1.787109375, -1.0986328125, 0.75048828125, -0.72900390625, -2.478515625, -1.41796875, -0.2156982421875, 0.9970703125, -0.464599609375, 1.84375, -1.3564453125, 0.446044921875, 0.1778564453125, 1.25, -0.059844970703125, -1.1611328125, -0.9833984375, -0.41552734375, 1.7958984375, 0.1341552734375, 0.490966796875, -0.2073974609375, 0.7900390625, 0.28271484375, 0.859375, -0.110107421875, -2.271484375, -0.4384765625, -0.08544921875, 0.1845703125, -0.31689453125, -0.040069580078125, 1.447265625, -1.169921875, 1.494140625, -1.080078125, -0.55615234375, -1.2646484375, -0.08642578125, -0.344970703125, -0.12261962890625, -0.1500244140625, -1.0771484375, 0.58837890625, 1.0703125, -0.0167694091796875, 1.0205078125, 0.86083984375, -1.3212890625, -0.06610107421875, -0.0738525390625, -1.591796875, -1.8193359375, 0.3193359375, -1.3544921875, 0.873046875, -1.3251953125, 0.0487060546875, 1.1435546875, 0.390380859375, -0.88671875, 0.02935791015625, 0.1734619140625, -1.1962890625, 0.98046875, 1.0361328125, 0.81884765625, 2.287109375, 0.779296875, -0.130615234375, 0.98388671875, 1.498046875, 1.53125, -0.10791015625, 1.1025390625, -0.313720703125, -0.75048828125, 1.48828125, 0.82666015625, -0.445068359375, 1.126953125, -1.3681640625, -0.625, -0.2049560546875, -0.25, 1.1943359375, 1.2451171875, -1.19921875, 2.5625, -0.80322265625, -0.1617431640625, -0.234130859375, 0.35595703125, 0.166015625, 0.057098388671875, 1.12109375, -0.4384765625, -0.76513671875, 0.009063720703125, -0.853515625, 0.00887298583984375, 1.4033203125, -0.3896484375, 0.09478759765625, -0.51171875, -0.42138671875, 0.5205078125, -1.021484375, 0.6005859375, -0.07916259765625, -0.0916748046875, 0.75830078125, 0.95458984375, 0.541015625, -0.92578125, 0.28369140625, 0.5087890625, 0.1251220703125, -0.247314453125, -1.6875, 0.49609375, -0.60595703125, 0.8076171875, -0.0010004043579101562, 1.5576171875, -0.8505859375, -0.84619140625, -0.283935546875, 0.05609130859375, -1.8603515625, 0.55517578125, 0.428466796875, 0.72412109375, 0.7021484375, 0.1397705078125, -0.2060546875, 0.8974609375, -0.83642578125, 0.310302734375, 0.388427734375, -0.5234375, 0.85009765625, 0.414794921875, -0.50634765625, 1.185546875, -0.8017578125, -0.6083984375, -1.75390625, 0.459228515625, -0.662109375, -1.552734375, 0.07733154296875, -0.63623046875, 0.031890869140625, -0.70263671875, -0.78369140625, 0.425537109375, 0.269287109375, -0.85302734375, 0.4775390625, -1.5439453125, -1.2509765625, -1.013671875, -0.1480712890625, -0.82666015625, -0.30810546875, 0.33935546875, 0.447509765625, -0.814453125, 1.505859375, 0.775390625, -0.7158203125, 0.78173828125, -0.3330078125, 0.4228515625, -1.3046875, 0.78564453125, -0.126953125, 0.97412109375, 0.935546875, 1.3662109375, -0.50390625, -1.3837890625, -0.11798095703125, 0.71630859375, 0.0295562744140625, -1.2060546875, 2.1015625, -0.44677734375, -1.3486328125, -1.5537109375, 0.3671875, -0.348876953125, 0.14453125, 1.7900390625, -0.74755859375, 1.1884765625, -0.11236572265625, -0.2261962890625, 0.285400390625, 1.96484375, -0.3076171875, 1.00390625, -1.150390625, -0.98046875, -0.7802734375, 0.365478515625, -0.7158203125, -0.89013671875, -1.529296875, -0.348876953125, 1.9267578125, -0.68115234375, 0.6396484375, 0.60791015625, -1.609375, -0.44775390625, -0.64794921875, -0.76513671875, 1.984375, -0.83642578125, 0.94091796875, -0.1395263671875, 0.59814453125, -0.119873046875, 0.84326171875, -1.69140625, 1.0302734375, -0.2822265625, 0.77099609375, 1.79296875, 0.1007080078125, -1.6337890625, -1.693359375, 0.8876953125, -0.8984375, -1.6943359375, 0.06597900390625, -0.0216827392578125, -0.1534423828125, 0.1370849609375, -1.7470703125, 0.744140625, -1.2578125, -0.200439453125, 0.6279296875, -0.56494140625, 0.278076171875, 0.396240234375, -0.0880126953125, -1.0869140625, 0.0312042236328125, 1.1220703125, 1.0966796875, -0.6376953125, -1.34765625, 0.71826171875, 0.422119140625, -1.1044921875, -0.259521484375, -1.5751953125, 0.38330078125, 0.03497314453125, -1.193359375, 0.3818359375, -1.1201171875, -0.1356201171875, -0.1259765625, 1.0107421875, -1.9873046875, -0.1724853515625, 0.021240234375, 0.95703125, 0.048736572265625, -1.7412109375, -2.080078125, -1.7568359375, 0.5654296875, 1.33984375, 1.7333984375, 0.0428466796875, -0.53271484375, -1.375, 0.666015625, 0.3603515625, 1.2578125, -1.3955078125, 0.67529296875, -0.483154296875, 1.166015625, -0.41015625, 0.261474609375, -1.5068359375, 0.10186767578125, -0.1737060546875, -0.86181640625, -0.57470703125, -0.7666015625, -0.93115234375, -0.61669921875, -0.07440185546875, 1.26953125, -0.80810546875, 1.783203125, -1.0341796875, 1.1962890625, 0.7470703125, -0.3916015625, 0.2103271484375, 0.54736328125, 0.89404296875, -0.8251953125, 0.79052734375, -1.9287109375, -0.0272674560546875, -0.18798828125, 1.1728515625, 0.22509765625, 0.2362060546875, -0.1993408203125, -0.57568359375, 0.196044921875, 0.308837890625, 0.1424560546875, 0.578125, -0.1767578125, -1.51171875, -1.1611328125, -0.321044921875, -1.1728515625, 0.84033203125, -0.112548828125, 0.6982421875, -0.0031528472900390625, 0.5439453125, -0.8564453125, -0.2320556640625, 1.9326171875, 0.142333984375, -1.3857421875, 0.0113372802734375, -1.0869140625, 1.4619140625, -1.748046875, 1.1630859375, -0.36962890625, -0.76318359375, 0.7158203125, 0.78125, -0.26708984375, -0.090576171875, 0.239013671875, -0.28857421875, 1.2724609375, -1.275390625, -0.54345703125, 1.033203125, -0.490234375, 0.04132080078125, 0.95068359375, 0.1806640625, 1.005859375, -0.10137939453125, -1.263671875, 0.09588623046875, 0.4013671875, 0.1363525390625, -0.4130859375, -0.196533203125, 0.40283203125, 2.4140625, -0.51708984375, 0.5732421875, 2.52734375, -1.1884765625, 2.236328125, -0.80029296875, 0.83837890625, 1.0771484375, 0.33740234375, -0.484375, 1.7431640625, -0.25244140625, 0.63134765625, -0.274658203125, 0.7783203125, 1.4873046875, 1.318359375, -0.8447265625, 1.4306640625, -0.391357421875, 0.85498046875, -0.90771484375, -0.53369140625, -0.5751953125, -1.2158203125, 0.348388671875, -1.9892578125, -1.46484375, -0.947265625, -0.861328125, 0.70947265625, -0.87646484375, -1.7548828125, -1.5400390625, -0.367431640625, 1.6572265625, 0.59130859375, 0.68359375, -0.309814453125, -0.5400390625, -0.970703125, -1.6787109375, -0.315673828125, -0.77587890625, -1.6962890625, 0.1697998046875, -0.7802734375, -0.64013671875, -0.29150390625, 0.9873046875, -0.7861328125, -0.94482421875, -0.351318359375, 0.869140625, 0.472900390625, 2.48046875, 0.1416015625, 1.3544921875, -0.267822265625, -0.8984375, 0.77197265625, 0.09918212890625, -0.72119140625, -0.6201171875, 0.94921875, -0.1026611328125, -0.73486328125, -0.4072265625, 0.80908203125, 1.091796875, -0.036163330078125, -1.30859375, -0.4833984375, 1.3798828125, 0.8017578125, 0.7646484375, -1.615234375, 0.441650390625, -0.330078125, 1.46875, -1.015625, 0.374267578125, -0.1334228515625, 0.291259765625, -0.482177734375, 0.2418212890625, 0.1689453125, 0.1658935546875, -0.908203125, -0.400390625, 0.2071533203125, 0.2161865234375, -0.18310546875, -0.43798828125, 0.88623046875, -1.5322265625, 0.5869140625, 0.360107421875, 0.413818359375, 0.197265625, 1.23046875, -1.3779296875, -0.92578125, -0.412109375, 1.7529296875, -0.66064453125, 1.2568359375, 0.84033203125, 0.50048828125, -0.340576171875, 2.56640625, -0.2275390625, -1.751953125, -0.90771484375, 0.269775390625, -1.6240234375, 0.2734375, -0.0936279296875, -2.373046875, -1.2236328125, -0.42333984375, 1.009765625, -0.21142578125, -1.576171875, 1.7333984375, 0.239013671875, -1.7060546875, 0.29150390625, -0.55859375, -0.27490234375, -0.16162109375, 0.7685546875, 0.76806640625, -0.6962890625, 1.0166015625, -0.5712890625, -0.64697265625, 1.365234375, -1.6220703125, -0.484619140625, 1.84375, -0.1710205078125, -0.9833984375, 1.1337890625, 0.91455078125, 0.83642578125, -0.029083251953125, 0.71728515625, 0.400390625, 0.128662109375, -0.47119140625, -1.0810546875, 0.3466796875, -1.0888671875, -1.375, -0.016571044921875, -0.85400390625, -0.53125, -0.1055908203125, -0.5068359375, 0.73876953125, 1.5146484375, -0.6611328125, -0.71826171875, 0.46142578125, 2.33984375, -0.71826171875, 0.032073974609375, -1.2607421875, -0.303466796875, -0.1087646484375, 0.47021484375, -0.8896484375, 1.74609375, -0.43603515625, -0.319091796875, 0.23779296875, -0.7578125, 1.2626953125, -1.8984375, -1.0654296875, 0.51953125, -0.39453125, 1.7880859375, -0.34765625, -0.385986328125, 0.9716796875, -0.9130859375, -0.285400390625, 0.8203125, 1.162109375, -0.0418701171875, 0.3837890625, 0.24853515625, 0.814453125, 0.3193359375, -1.083984375, -0.441162109375, 1.1494140625, -1.5947265625, -0.35888671875, -0.4521484375, -0.294921875, -0.97802734375, -0.415283203125, -0.72998046875, 1.1083984375, 0.340576171875, -0.6240234375, -0.10345458984375, 3.3984375, 0.8935546875, -1.359375, -0.02716064453125, 0.5107421875, -1.0185546875, 0.484130859375, 0.325439453125, -0.3291015625, 2.505859375, -0.56982421875, -0.96337890625, 0.60791015625, -0.97021484375, -0.41650390625, -0.356201171875, -0.51318359375, 0.4365234375, -0.01245880126953125, 1.4365234375, 0.5498046875, 0.92822265625, -1.4970703125, -0.1824951171875, -1.6337890625, 0.1710205078125, 0.312744140625, -0.347900390625, 0.7763671875, 0.68994140625, -1.1669921875, -0.5615234375, -0.92578125, -0.95263671875, -0.89794921875, -0.984375, 1.8623046875, -0.63232421875, 0.1273193359375, 0.74169921875, 1.0380859375, 0.0232391357421875, -0.498046875, -0.2061767578125, 0.3798828125, 0.36328125, -0.094970703125, 0.50390625, -0.62451171875, 0.93505859375, 0.3349609375, 1.9208984375, -1.4091796875, 0.09661865234375, 0.45458984375, -0.292236328125, -1.6728515625, 1.2353515625, 1.203125, 0.173828125, -1.615234375, -0.75048828125, -0.6162109375, -0.376708984375, 1.0576171875, -1.95703125, -1.353515625, -0.1661376953125, 0.482421875, 0.37255859375, 0.5927734375, -0.9912109375, -1.0546875, -1.2392578125, -1.47265625, 0.033172607421875, 0.1129150390625, 2.484375, 1.42578125, 0.68017578125, -0.87841796875, 1.458984375, 1.716796875, 0.359130859375, 1.3095703125, 0.50537109375, 2.458984375, 1.2998046875, 1.015625, -0.441162109375, 2.5078125, 1.2705078125, 0.11444091796875, 1.0224609375, -1.2548828125, 0.436767578125, -0.6923828125, -1.5947265625, -0.3388671875, 0.990234375, 1.5673828125, -1.15625, -0.70947265625, -2.263671875, 0.591796875, -1.228515625, 0.1864013671875, 0.8525390625, 1.625, -0.2080078125, 0.41064453125, -0.261962890625, 0.04327392578125, -1.7607421875, -0.364501953125, -0.5107421875, -0.587890625, -1.33984375, 0.482666015625, -0.43310546875, 2.037109375, 0.345703125, -0.394287109375, -0.41455078125, -0.137939453125, -1.263671875, -0.1214599609375, -0.3017578125, 0.1661376953125, 1.04296875, 0.77099609375, -0.1134033203125, -1.1015625, -1.244140625, 0.67333984375, -0.1395263671875, -0.82666015625, -0.9013671875, -0.2147216796875, -0.68115234375, -0.03863525390625, -0.8076171875, 1.1015625, -0.383544921875, 0.310791015625, -0.3837890625, -0.71435546875, -0.2098388671875, -0.81787109375, 0.88134765625, -0.95263671875, -0.1536865234375, 1.25390625, -0.237060546875, -0.485595703125, 1.201171875, -0.139404296875, 1.0185546875, -0.004390716552734375, -0.9169921875, 0.3212890625, 0.46533203125, -0.68701171875, -0.342041015625, -0.050262451171875, -1.0146484375, -0.71875, 0.045867919921875, -0.26513671875, 0.55078125, -1.5625, -1.884765625, -0.544921875, 0.244384765625, -1.349609375, -0.26513671875, -0.32568359375, -0.576171875, -0.0721435546875, -0.450927734375, 2.30078125, -0.88232421875, -0.83154296875, -1.3779296875, -1.513671875, 0.67822265625, -0.69189453125, -1.654296875, 0.1650390625, 0.55517578125, -0.9130859375, -0.72265625, 1.2548828125, -0.263427734375, 0.368408203125, -1.625, -1.443359375, -0.429443359375, 1.1279296875, -2.55078125, -0.5634765625, 0.69677734375, -0.51953125, -0.11663818359375, 1.447265625, 0.440185546875, 0.40185546875, -1.0546875, 1.92578125, 0.69091796875, -0.68359375, -1.0546875, -0.2200927734375, 0.1689453125, -0.2113037109375, -0.1192626953125, 1.4462890625, 0.111572265625, -1.1494140625, 0.81494140625, 0.54248046875, 0.334228515625, 1.4140625, -1.16796875, 0.27490234375, -0.32373046875, -0.478271484375, -1.5693359375, -0.9921875, 0.12286376953125, -0.447509765625, 0.0640869140625, -0.0594482421875, -0.2437744140625, 0.10888671875, -0.02532958984375, -0.466796875, 0.54296875, 0.050201416015625, 1.537109375, -0.452392578125, 0.615234375, -1.2529296875, -0.6640625, 0.9169921875, 0.311767578125, -0.62109375, 1.2158203125, 0.1822509765625, 2.23828125, -1.2822265625, 0.72216796875, -0.004749298095703125, 0.79248046875, 0.3681640625, -0.6630859375, 0.2227783203125, -2.45703125, 0.7529296875, -0.327880859375, 0.2120361328125, -0.83056640625, 0.52587890625, 0.48583984375, -0.246826171875, 0.0789794921875, 0.88134765625, 0.2607421875, -0.3857421875, 1.904296875, -1.169921875, -1.04296875, -2.62109375, 0.274169921875, -0.0662841796875, 1.421875, -0.685546875, -1.91015625, -2.958984375, -0.1727294921875, -0.318603515625, -0.654296875, -1.0859375, 0.25146484375, 0.326171875, -0.47998046875, -1.212890625, -1.0302734375, 0.86083984375, 0.36181640625, -0.693359375, -0.26416015625, 1.0205078125, 0.382080078125, -1.2939453125, 0.44775390625, 0.83154296875, -3.115234375, -0.44873046875, -0.266845703125, 1.9453125, 0.86474609375, -0.42626953125, 1.3427734375, 1.30078125, -1.044921875, 0.5048828125, -0.56103515625, -1.4462890625, -1.35546875, 0.2470703125, 0.239013671875, 0.2469482421875, -0.52490234375, -0.47412109375, 0.362060546875, 0.239990234375, -1.01171875, -1.1796875, 0.058746337890625, -0.7392578125, -1.166015625, 0.64794921875, -0.262451171875, 1.3232421875, 0.58251953125, 0.61181640625, 1.1953125, -0.07208251953125, -0.87646484375, -0.229736328125, 0.697265625, -0.52734375, -0.6640625, 1.94921875, 0.74755859375, -0.77197265625, 1.8125, 2.861328125, 0.1937255859375, -0.2032470703125, 0.242919921875, -0.79248046875, -1.3330078125, -1.2724609375, -1.12890625, 1.03515625, -0.888671875, -0.004009246826171875, -0.12237548828125, 0.89404296875, 0.2529296875, 0.140380859375, 0.007724761962890625, -1.1396484375, 0.1883544921875, 0.9794921875, 0.75439453125, -1.0205078125, -0.257568359375, 0.91943359375, 1.138671875, -0.1783447265625, 2.794921875, -0.044647216796875, -1.34375, -0.291748046875, 0.52978515625, 0.24267578125, 0.456298828125, -0.03363037109375, -0.272705078125, -1.01171875, 0.90576171875, -1.7294921875, 0.88916015625, 0.95361328125, 0.73583984375, 1.4111328125, 0.00801849365234375, 2.048828125, 0.06390380859375, -0.580078125, 0.9228515625, 0.485107421875, 0.305419921875, -0.5673828125, -0.38720703125, -0.99365234375, 0.96630859375, -1.4150390625, 0.05609130859375, 1.447265625, -0.6962890625, -1.162109375, -0.78369140625, 0.51904296875, -0.55517578125, -1.4755859375, -1.11328125, -0.04986572265625, -1.2802734375, -0.91845703125, -0.44189453125, 0.85791015625, 1.5390625, 0.6669921875, 0.01092529296875, -0.29833984375, -0.1265869140625, 0.12249755859375, -0.87646484375, 1.2412109375, 0.186279296875, -0.447998046875, 0.1500244140625, 0.98486328125, 0.428466796875, -1.3740234375, -0.11834716796875, -0.4609375, -0.509765625, 0.81494140625, 0.052398681640625, -1.0302734375, -1.05859375, 0.213623046875, 0.5732421875, -0.71923828125, -0.51220703125, -0.25439453125, -1.44921875, 0.60791015625, 2.05078125, 0.397216796875, 0.1282958984375, 0.4912109375, 0.81787109375, 0.62451171875, -0.97705078125, 0.59130859375, -1.166015625, -1.3349609375, -0.2156982421875, -0.68115234375, 0.0213775634765625, -2.908203125, 0.11669921875, -1.4794921875, -0.52880859375, -0.77734375, -1.40625, 2.025390625, 0.397216796875, -0.4267578125, 0.9208984375, 1.2783203125, -1.2919921875, -0.7529296875, -0.69775390625, -0.2568359375, -1.9208984375, 0.47021484375, -0.552734375, 0.46337890625, 0.353271484375, -1.42578125, 0.1668701171875, 0.4990234375, -0.1629638671875, 1.0703125, -0.78125, -1.68359375, -0.12396240234375, -0.021820068359375, -0.2406005859375, -0.041717529296875, -0.650390625}; - -static const __fp16 Bpack_dram[4096] - __attribute__((section(".dram"), aligned(64))) = {0.1588134765625, 1.8232421875, -0.055389404296875, -0.72265625, 1.05078125, 0.64697265625, -0.1920166015625, 2.998046875, -0.1162109375, 0.25439453125, 1.126953125, 0.402099609375, -0.181396484375, 0.379638671875, -1.310546875, -0.90234375, 0.185546875, -0.041595458984375, 1.16015625, 0.057281494140625, -0.422607421875, -0.22900390625, 0.8681640625, 1.82421875, 0.66162109375, -1.4345703125, 0.63720703125, 0.923828125, -0.1268310546875, -0.26708984375, -1.46875, -1.0166015625, 1.025390625, 0.355712890625, 0.7548828125, -0.378662109375, 3.349609375, -0.5498046875, 0.79150390625, 0.08575439453125, 1.5166015625, 0.122314453125, 0.62841796875, 1.2353515625, -1.396484375, 1.7373046875, 1.033203125, 0.67529296875, -0.6015625, -0.3837890625, 0.10321044921875, -0.40576171875, -1.294921875, 0.32080078125, 2.1484375, -0.00826263427734375, 0.08502197265625, 0.66162109375, 1.525390625, -0.273681640625, -0.455322265625, 0.1683349609375, 1.2890625, -0.338134765625, -1.1455078125, 0.57861328125, -0.08087158203125, 1.1494140625, -0.056640625, -0.189208984375, -0.82763671875, 0.305419921875, 0.384033203125, 0.56591796875, -1.427734375, -1.103515625, 0.0352783203125, -1.337890625, -1.9599609375, -1.5654296875, 1.3349609375, 1.4345703125, 0.59033203125, 2.39453125, 1.5498046875, 0.251708984375, 0.045166015625, -0.1600341796875, -0.77490234375, -0.6845703125, 0.03350830078125, 0.0870361328125, 0.6533203125, -2.49609375, -0.047607421875, 0.91357421875, -0.9248046875, -0.97509765625, 0.43896484375, -0.49755859375, 0.759765625, -0.266845703125, -1.3623046875, 0.045928955078125, -0.13916015625, -1.2939453125, -1.330078125, 0.0291290283203125, -1.265625, 0.00833892822265625, 1.33203125, 1.3291015625, -1.37109375, -3.103515625, 0.3095703125, 0.39453125, 0.171142578125, -0.12353515625, -0.78857421875, -0.84814453125, 1.224609375, 1.59765625, 1.08984375, 0.1492919921875, -0.32177734375, 0.5625, 1.56640625, 0.78271484375, -0.77392578125, 0.5869140625, 0.732421875, -1.1474609375, 0.63720703125, 0.09429931640625, 0.484375, 0.79931640625, 0.06060791015625, 1.8828125, 0.1729736328125, 0.89208984375, 0.69873046875, -1.2705078125, -2.064453125, 0.681640625, -1.3427734375, 0.931640625, -1.9853515625, -0.734375, -1.2099609375, 0.0103302001953125, 0.326416015625, 0.72900390625, -0.20556640625, -1.056640625, -2.810546875, -0.2249755859375, -0.53125, 0.1351318359375, 0.763671875, -0.18359375, 1.720703125, -0.194580078125, -0.268798828125, 0.76904296875, -2.8125, -1.9453125, 0.097412109375, 0.1875, -0.0726318359375, -1.16796875, 0.68017578125, 0.9482421875, -0.386474609375, -0.309326171875, -0.0258636474609375, 0.92626953125, 0.853515625, -1.5830078125, 0.301513671875, 0.1641845703125, 0.3173828125, -1.365234375, 0.2685546875, -2.353515625, -0.5888671875, -0.6875, 1.2060546875, -0.412353515625, -0.21142578125, 0.85107421875, 1.234375, -0.40087890625, -1.7802734375, 0.52001953125, -1.3173828125, -2.109375, 0.23828125, -0.50927734375, 0.47021484375, -0.164306640625, 0.9423828125, 0.59619140625, 2.091796875, -1.1865234375, -2.3046875, -1.33984375, 0.6845703125, 0.217529296875, 0.44580078125, 1.341796875, -1.056640625, -0.451904296875, 0.444580078125, -0.90478515625, 0.200439453125, 1.0263671875, -1.9892578125, 0.131591796875, -0.73486328125, -1.359375, -1.22265625, 0.1849365234375, 0.61767578125, -1.1376953125, -0.5830078125, 1.2470703125, 0.7646484375, 2.740234375, 0.5986328125, 0.183837890625, 0.363037109375, 0.2457275390625, 1.705078125, 0.034423828125, 1.0556640625, -1.7431640625, 1.326171875, -1.150390625, 0.1802978515625, -0.8515625, 0.44091796875, 0.377685546875, -0.28515625, 0.2158203125, -0.41650390625, 1.48046875, 2.193359375, 0.63916015625, 0.55078125, -0.619140625, -1.2978515625, 0.90673828125, -0.360107421875, -1.9599609375, -0.271484375, -2.0703125, -2.072265625, 0.81591796875, 2.015625, 1.7841796875, -1.9619140625, 0.71435546875, 0.261962890625, 1.45703125, -1.2685546875, -1.48828125, -0.414306640625, 0.0791015625, -0.48779296875, 0.64990234375, -1.3779296875, -0.84375, -1.0830078125, -0.0887451171875, 0.767578125, 2.712890625, -1.564453125, 0.38525390625, -0.14794921875, -1.4951171875, -1.244140625, 0.658203125, -0.2294921875, -0.5146484375, -0.2227783203125, 0.61279296875, -0.90869140625, -0.351318359375, -1.4189453125, 0.580078125, -0.135498046875, -1.6328125, -1.853515625, -0.1767578125, -0.37451171875, 0.491455078125, -0.6298828125, -0.27783203125, 1.8134765625, -0.428955078125, 1.0224609375, 0.054534912109375, -1.75, 0.29931640625, 1.1162109375, 0.0189666748046875, -0.88916015625, -0.70068359375, -0.9462890625, -1.337890625, 0.90283203125, -0.427978515625, -0.30126953125, -0.29638671875, -1.6533203125, -0.71044921875, -0.433349609375, -0.349853515625, -0.392333984375, -0.66748046875, 0.1787109375, -0.0845947265625, -1.978515625, -0.974609375, -1.2685546875, -0.1729736328125, -0.1240234375, -0.63427734375, 0.97998046875, -0.390380859375, 1.080078125, 1.32421875, -0.998046875, 0.826171875, 1.3642578125, 0.662109375, -0.71826171875, -0.74365234375, -0.69873046875, 0.255859375, -1.2451171875, -0.282470703125, -1.0830078125, -0.84423828125, 1.3447265625, -1.0908203125, -1.0107421875, -0.923828125, -0.83935546875, 1.4443359375, -1.4541015625, -1.4921875, 0.52587890625, 1.0693359375, 1.4208984375, -0.64013671875, 0.36279296875, 0.1793212890625, 1.0859375, 0.8251953125, 0.08233642578125, 1.1689453125, 1.0888671875, 1.2294921875, 0.1263427734375, -0.943359375, 1.7197265625, -1.861328125, -1.466796875, 0.66064453125, -0.1412353515625, -0.83935546875, 0.5966796875, -0.35498046875, -1.3056640625, -0.269287109375, -0.4765625, 1.0751953125, -0.8056640625, -0.9052734375, -0.8310546875, -0.41650390625, 1.869140625, -0.833984375, -0.1700439453125, -0.38037109375, -0.1373291015625, 0.61767578125, -0.69970703125, 0.36767578125, 0.313720703125, 1.0224609375, -0.62109375, -0.658203125, -0.32275390625, 0.63134765625, -0.0186920166015625, 0.626953125, -2.126953125, -0.4736328125, -0.445068359375, 0.1773681640625, -0.8037109375, -0.213623046875, -0.33740234375, 1.234375, 0.353515625, 0.418212890625, 1.1689453125, 0.7744140625, -0.040283203125, -0.34033203125, 0.7109375, 1.2314453125, -0.016998291015625, 0.38232421875, -0.80908203125, 1.0478515625, 0.3701171875, 0.0362548828125, -0.888671875, -0.52880859375, 1.5224609375, 1.234375, -0.0843505859375, 0.224609375, -1.46484375, 0.14501953125, -0.1812744140625, -0.60595703125, 0.724609375, -1.849609375, -0.428955078125, 1.6025390625, 1.3134765625, -0.302978515625, 1.81640625, 0.206298828125, 1.3310546875, -1.0283203125, 0.6640625, -0.081298828125, -0.81689453125, 0.310546875, 0.93408203125, -0.09820556640625, -1.7783203125, -0.97900390625, -1.4248046875, 1.2265625, 1.125, 0.97509765625, -0.7275390625, 0.947265625, -0.87353515625, 1.537109375, 1.4560546875, -0.397216796875, -0.0736083984375, 0.72119140625, -0.240478515625, -0.1463623046875, -0.272705078125, -0.8955078125, -0.3525390625, -0.4521484375, 1.0908203125, -0.280517578125, -1.1083984375, -1.1884765625, 1.7109375, 1.833984375, 0.388671875, -0.2005615234375, -1.5419921875, 0.9658203125, 0.169677734375, -0.8603515625, -0.264892578125, 0.025238037109375, -1.1884765625, 0.10650634765625, 0.489501953125, -1.5107421875, 0.306640625, 0.2276611328125, 0.52197265625, 0.58203125, 0.198974609375, -0.3212890625, 1.2275390625, -0.51513671875, 1.32421875, 1.109375, 1.16796875, 1.0048828125, 0.094482421875, 0.1763916015625, -1.57421875, -0.27197265625, -0.455078125, -1.7041015625, -0.17041015625, -1.759765625, 0.00963592529296875, -0.6416015625, -0.94970703125, -0.451171875, -0.48828125, -0.310302734375, -0.970703125, 0.97509765625, -1.4619140625, -0.13525390625, 0.378173828125, 0.60986328125, 0.72705078125, 0.72900390625, -0.0604248046875, -0.99072265625, 0.07489013671875, -1.64453125, 0.2161865234375, 1.796875, -0.53955078125, -0.202880859375, -0.4560546875, 0.0017175674438476562, -0.55419921875, 0.10491943359375, 0.10675048828125, 0.9306640625, 0.8466796875, 0.525390625, -1.4140625, -0.006084442138671875, 0.0286865234375, -0.65625, 0.955078125, -0.59228515625, 1.70703125, 0.40380859375, -1.130859375, 0.9072265625, -0.290771484375, -0.051605224609375, -0.1270751953125, 0.1702880859375, -1.4462890625, -0.142578125, 0.97509765625, 1.0849609375, 0.30419921875, -1.51953125, 1.6044921875, -1.712890625, 2.6640625, -0.99658203125, -0.7431640625, -0.181396484375, -0.80224609375, -0.80322265625, 1.203125, -1.794921875, -1.6455078125, 1.06640625, -1.2509765625, 0.83642578125, -1.8076171875, 0.78955078125, -0.1707763671875, 0.09808349609375, 2.087890625, -0.452880859375, 0.26708984375, 0.69677734375, -0.537109375, 1.974609375, -0.268310546875, -0.6640625, 1.041015625, 0.68896484375, -0.375732421875, 1.36328125, -0.5703125, 0.434326171875, 0.2408447265625, -1.66796875, -0.477783203125, 0.560546875, 0.265625, -0.53515625, -0.88037109375, -1.3798828125, -1.6708984375, 0.54541015625, 0.052276611328125, -2.70703125, 0.020782470703125, -0.453125, -0.0277557373046875, 0.89306640625, -1.2802734375, -0.60205078125, -0.04827880859375, 0.21630859375, 0.62939453125, -0.78857421875, -0.55615234375, -0.57080078125, 0.443115234375, 0.93017578125, 1.2841796875, -2.6015625, -0.4892578125, 1.7958984375, 1.5771484375, 0.41943359375, 0.6474609375, -0.6337890625, 0.66064453125, 0.56689453125, 0.27880859375, -0.63720703125, -0.6650390625, 1.373046875, 0.2119140625, -0.3486328125, 1.0634765625, 0.00933837890625, 1.720703125, -1.0595703125, 0.03369140625, 0.64013671875, 0.92431640625, 0.0174407958984375, -0.1583251953125, -0.382080078125, -1.2080078125, -0.8115234375, 0.09503173828125, 0.7109375, 1.6826171875, 0.6669921875, 1.4775390625, 1.130859375, 1.0537109375, 0.1337890625, 1.6005859375, -1.1005859375, 1.0234375, -0.8466796875, -0.52978515625, 0.51220703125, -0.54541015625, -0.63525390625, 1.5498046875, 0.06488037109375, -0.159912109375, 0.6201171875, -1.6015625, 1.091796875, 0.1268310546875, -0.3310546875, -0.434814453125, 0.01107025146484375, -1.7451171875, -0.771484375, 0.60595703125, 0.384033203125, -1.40625, -0.268310546875, -0.114013671875, -1.2158203125, -0.48486328125, 0.274169921875, -1.07421875, 0.328369140625, -1.8671875, -2.18359375, -1.263671875, -0.7509765625, -0.9130859375, -1.1982421875, 1.1064453125, -1.2763671875, 1.01953125, -0.368896484375, 0.287109375, -0.247802734375, 0.471435546875, -2.47265625, 0.39404296875, 0.3876953125, 2.11328125, -0.66259765625, 0.59912109375, 0.0004153251647949219, -0.443359375, -1.029296875, 1.3828125, 0.962890625, -0.0540771484375, 0.69482421875, 0.0906982421875, -0.280029296875, -0.0523681640625, -0.343505859375, 0.00811767578125, -0.2215576171875, 1.900390625, 0.83935546875, 0.59326171875, -1.248046875, 1.0751953125, -0.2215576171875, -0.8173828125, 0.92138671875, 0.81494140625, -0.1334228515625, -0.362060546875, 0.36328125, -0.76171875, 0.84521484375, 0.9951171875, 0.775390625, -1.3623046875, 0.020050048828125, -0.162109375, 0.59423828125, -0.87939453125, 0.1312255859375, -0.19384765625, 0.16845703125, -0.71337890625, -1.9169921875, -1.5185546875, -0.98095703125, 0.181884765625, -0.85693359375, -0.1434326171875, -0.5966796875, 1.0322265625, 0.17236328125, -0.395263671875, -0.18310546875, 0.005462646484375, -1.2412109375, 0.8525390625, 0.218994140625, -0.6611328125, 2.017578125, -0.0880126953125, 0.382568359375, -0.2734375, 1.0966796875, -1.9921875, -0.497314453125, 0.62939453125, 1.3515625, 0.5234375, -0.49267578125, -0.94482421875, -0.72998046875, 0.1083984375, 1.017578125, 1.8994140625, -0.170166015625, 1.2587890625, 0.31494140625, -0.91552734375, -0.5673828125, 0.49169921875, 0.449951171875, 0.6162109375, 1.01171875, 0.22998046875, 0.9130859375, -0.1258544921875, -1.4951171875, -0.328125, 0.054718017578125, -0.830078125, 2.041015625, 0.5634765625, 1.64453125, 1.021484375, -0.548828125, -0.36669921875, -0.90576171875, -0.07470703125, -1.1298828125, -0.0509033203125, 0.00830078125, 1.482421875, 1.2138671875, -0.17041015625, -0.697265625, 0.61328125, -0.58984375, -0.9541015625, 0.311767578125, 1.3857421875, -1.5146484375, -0.164306640625, -1.9150390625, -0.71875, 1.4501953125, -0.2274169921875, -1.8544921875, -1.125, -0.6416015625, 1.4833984375, 0.427001953125, -1.0498046875, -0.11053466796875, 0.41845703125, -0.2147216796875, -0.32275390625, -1.2109375, -1.0546875, 0.39599609375, 1.126953125, 0.05841064453125, -0.33203125, 0.26513671875, 1.0498046875, 0.10693359375, -0.91259765625, 0.5537109375, 0.6611328125, 1.337890625, 0.599609375, 1.046875, -0.011627197265625, -0.0203399658203125, -0.64697265625, 0.208984375, 0.1202392578125, -0.7578125, -0.0751953125, -0.51123046875, -0.79150390625, -1.83203125, 1.01953125, -1.5625, 1.3369140625, -1.2802734375, -0.6875, 0.82958984375, -0.8173828125, -0.193115234375, -0.330078125, 0.62646484375, -1.2548828125, -0.81689453125, -1.091796875, 0.6796875, 0.48046875, -0.92138671875, 1.224609375, 0.5419921875, -0.92822265625, 0.52978515625, -1.2841796875, 0.77587890625, 0.2021484375, -0.29296875, 0.061004638671875, -0.02813720703125, 1.248046875, 1.5087890625, -0.0271453857421875, 1.1904296875, -1.779296875, 0.01171875, 0.3447265625, -0.34765625, -1.45703125, -0.84814453125, -1.2900390625, -0.8408203125, -0.52978515625, 1.1572265625, -1.26171875, 2.080078125, -1.7138671875, -2.595703125, -0.7421875, -1.2646484375, -0.274658203125, 0.030242919921875, -0.195068359375, -1.267578125, 0.195068359375, 0.40869140625, -0.3779296875, 0.93798828125, -0.61376953125, 0.239501953125, -0.66845703125, 0.6865234375, -2.201171875, -0.1234130859375, -0.267822265625, -0.1085205078125, 1.390625, -0.43994140625, -0.5625, -1.587890625, -1.16015625, -0.254638671875, 0.89111328125, 0.09625244140625, 0.70947265625, 1.310546875, 0.205322265625, 0.10980224609375, 1.19921875, 0.36376953125, 0.2724609375, 0.2451171875, 0.192138671875, -0.2379150390625, -0.8798828125, -2.544921875, -0.2467041015625, -1.48828125, 1.5712890625, 0.452880859375, -0.7978515625, 0.71630859375, 0.068115234375, 0.94580078125, -0.46630859375, -0.01251983642578125, -0.3330078125, -0.495361328125, 0.0241546630859375, -1.419921875, 0.2188720703125, 3.310546875, 1.4267578125, -1.5966796875, 1.4638671875, -0.5703125, 0.6923828125, 2.748046875, 0.4853515625, 1.447265625, 0.80078125, -0.495849609375, 0.47998046875, -0.8798828125, 0.859375, -0.98583984375, 1.259765625, 1.3349609375, 0.304931640625, 0.58984375, 1.173828125, 0.277587890625, 0.379150390625, -0.225341796875, -1.380859375, -0.1763916015625, -0.84326171875, -0.85205078125, 1.3447265625, -0.658203125, -2.544921875, -1.228515625, -1.9423828125, -1.591796875, -0.228515625, -0.67431640625, -0.14501953125, -1.072265625, -0.038330078125, 0.068115234375, -1.5458984375, 0.372802734375, -0.54345703125, 1.3291015625, -1.11328125, 0.36767578125, 0.050811767578125, -0.34716796875, 0.30615234375, 0.50048828125, 0.08038330078125, 1.3662109375, -1.044921875, -0.958984375, 0.96923828125, -0.343505859375, -0.5322265625, 0.11602783203125, 0.159912109375, 0.432861328125, 0.57470703125, -0.2880859375, -1.0478515625, -0.98388671875, 0.310302734375, 1.607421875, 1.65234375, -0.80126953125, 0.384521484375, 0.9677734375, 0.0689697265625, 0.1236572265625, -0.72900390625, -1.0947265625, -0.219970703125, -0.325439453125, 0.63427734375, -1.4462890625, 1.095703125, -0.1771240234375, -0.492919921875, 1.80078125, -0.1385498046875, 0.64111328125, -2.560546875, -0.480712890625, 0.374267578125, -0.13330078125, -1.2197265625, -0.64111328125, 1.345703125, -1.0517578125, -1.599609375, 1.1494140625, -0.337158203125, 0.474609375, -0.260009765625, 0.27685546875, 1.1796875, -0.958984375, -1.13671875, 0.83544921875, 0.0814208984375, -0.6533203125, 0.11236572265625, 0.66064453125, 0.6123046875, -1.62109375, 0.385009765625, -1.75, 0.29296875, -0.391357421875, 0.08538818359375, -0.4482421875, 1.2421875, -1.462890625, 0.87890625, 1.62890625, -0.491943359375, -0.1229248046875, -0.53173828125, 0.53662109375, 0.62890625, -0.489501953125, -0.87060546875, 0.044769287109375, 0.223876953125, 2.07421875, 0.64990234375, -0.60546875, 1.10546875, 0.75732421875, -0.262451171875, -0.8662109375, 1.0341796875, 0.371337890625, 1.666015625, 0.31884765625, -0.06243896484375, 0.1478271484375, -0.5341796875, -0.1392822265625, 1.46484375, 0.303466796875, 1.875, -0.6748046875, -0.7412109375, -0.9814453125, 1.828125, 0.68798828125, 1.5703125, -0.64794921875, -0.39501953125, -0.7099609375, -0.63134765625, 1.994140625, -0.63232421875, 1.6044921875, -0.869140625, 0.9052734375, 0.56982421875, -1.6044921875, -0.03643798828125, -0.49267578125, -0.0667724609375, 0.896484375, -1.2978515625, -0.76708984375, 1.7548828125, 1.2509765625, -0.79150390625, 1.5615234375, 1.0498046875, 1.6962890625, -0.63427734375, -1.333984375, -0.60595703125, -0.61962890625, 0.03607177734375, -0.040740966796875, -1.7333984375, -2.92578125, -0.5087890625, -0.385498046875, 0.1842041015625, -0.603515625, -0.6435546875, 0.415283203125, 1.84375, 1.279296875, -0.1829833984375, 0.85498046875, 1.5029296875, 0.32177734375, 0.3076171875, 0.857421875, -0.316650390625, 0.7900390625, -1.1953125, 0.067138671875, -1.08203125, 0.1552734375, 1.02734375, 1.982421875, 0.67919921875, 0.99560546875, 0.91845703125, -1.8671875, -0.1468505859375, 0.400146484375, 0.14208984375, 1.439453125, 0.96142578125, -0.7861328125, 0.1468505859375, -0.6884765625, -0.55126953125, -0.681640625, 0.1591796875, -1.02734375, 0.875, 0.36328125, 1.7890625, 0.313720703125, 1.3154296875, -0.2191162109375, -2.240234375, 0.5654296875, 0.66259765625, -0.82275390625, 0.5166015625, 0.6669921875, 0.406982421875, 2.32421875, -1.0478515625, 2.279296875, 0.65869140625, -1.2861328125, -1.03515625, 1.3896484375, -0.454833984375, -0.276611328125, 0.1768798828125, -0.64990234375, 0.94384765625, 1.1767578125, -1.955078125, 0.7138671875, -0.417724609375, 0.650390625, 1.0908203125, 1.5673828125, -0.1783447265625, 3.01171875, -0.135986328125, -1.29296875, 0.40478515625, -0.85498046875, 0.9326171875, -0.73486328125, 0.1759033203125, -0.0247802734375, 0.96435546875, 1.611328125, -2.09375, 0.689453125, 0.39208984375, -0.9794921875, 0.63037109375, -0.263916015625, -0.708984375, 0.7861328125, 1.5390625, -2.134765625, 1.3212890625, -0.08477783203125, 1.1015625, 1.009765625, 1.787109375, 0.7998046875, -0.274658203125, -0.0241546630859375, -0.619140625, -0.99365234375, 0.98388671875, -0.9912109375, 0.6435546875, 1.9736328125, -1.76171875, -0.1832275390625, -0.798828125, 0.07861328125, 0.86865234375, 1.1474609375, 1.5029296875, 0.74755859375, -0.1976318359375, -1.1181640625, -0.023529052734375, 0.096435546875, 0.845703125, -1.169921875, 1.25, 0.1507568359375, -0.751953125, -0.28125, 0.95458984375, -0.33349609375, 0.67138671875, 0.29296875, 0.73681640625, 0.4453125, -0.5888671875, 0.44873046875, -0.86865234375, -0.0210113525390625, 1.5498046875, -0.2379150390625, 0.4130859375, 0.49072265625, 0.06707763671875, 1.1845703125, 0.0203399658203125, -0.83837890625, 1.150390625, -1.1337890625, -0.1805419921875, 0.892578125, 0.4814453125, 1.0947265625, -1.5712890625, 0.450927734375, -1.2744140625, 0.6142578125, 0.6455078125, -0.79052734375, -1.345703125, 0.62890625, -0.74853515625, -1.1611328125, -0.6845703125, 0.0909423828125, 0.2379150390625, 1.634765625, 0.50048828125, 0.9638671875, 0.89208984375, 0.79296875, 1.4931640625, -1.1904296875, -0.96533203125, -1.2041015625, 1.587890625, 1.017578125, -0.351806640625, -1.2578125, 0.74462890625, 1.2236328125, -0.96240234375, 0.68896484375, 0.4560546875, -0.17626953125, 1.0146484375, -1.251953125, -0.82763671875, -0.47607421875, -1.830078125, 0.8955078125, 0.2088623046875, -1.6435546875, -0.9501953125, -0.8798828125, 1.3759765625, 0.485595703125, -1.5947265625, 0.376708984375, 0.167724609375, 1.7353515625, 2.828125, -0.6953125, -0.04071044921875, -1.37890625, 0.7138671875, 0.6435546875, -2.099609375, -0.97705078125, 1.234375, 1.5126953125, -1.283203125, 1.845703125, 0.391845703125, 0.07989501953125, -0.43212890625, 0.479248046875, -0.98095703125, -2.703125, -0.826171875, 0.00421142578125, -1.189453125, -0.4365234375, 0.310546875, 0.66455078125, 0.0723876953125, 0.767578125, -1.1875, 0.411376953125, 0.89404296875, 0.7724609375, 0.73486328125, -1.8349609375, -1.201171875, 0.65380859375, -0.12451171875, -1.2421875, 0.7255859375, 0.03863525390625, 0.1453857421875, 1.255859375, 1.361328125, 0.189697265625, 0.050689697265625, -0.5849609375, 0.478515625, -0.3212890625, -2.0390625, 1.806640625, -0.72314453125, -0.059295654296875, 0.58154296875, -0.9580078125, -1.123046875, -0.41455078125, -0.2430419921875, -1.1572265625, 0.77490234375, -1.0859375, -2.365234375, -0.54296875, 1.39453125, 0.070068359375, 0.37890625, -0.1748046875, -0.318603515625, -1.15234375, 0.70166015625, -1.0673828125, -1.4716796875, 0.292236328125, 0.52197265625, 1.1650390625, 0.6533203125, 0.59912109375, 0.97900390625, 2.322265625, 0.86279296875, 0.137451171875, -1.44140625, -0.005718231201171875, -0.31494140625, 1.7412109375, -0.94140625, -0.87109375, -0.387451171875, 0.2271728515625, 0.341064453125, -0.404052734375, 0.1412353515625, -0.337890625, 0.2174072265625, 0.436279296875, 0.2205810546875, -0.130615234375, -0.301025390625, 1.9150390625, 0.96875, 1.1650390625, -0.69091796875, -1.2177734375, 0.7685546875, 1.5185546875, 2.279296875, -0.9482421875, 0.11016845703125, 0.48291015625, 0.16162109375, -0.1043701171875, 1.095703125, -1.119140625, -0.0582275390625, -0.34619140625, -1.6650390625, 0.34228515625, -1.02734375, -0.45751953125, 0.720703125, 1.5771484375, 0.654296875, -0.64306640625, -1.3212890625, -0.857421875, 1.0166015625, -0.232421875, 0.31494140625, 1.1328125, 0.294921875, -0.52197265625, -0.0207061767578125, 1.1875, 0.264404296875, -1.392578125, 0.423828125, -0.2568359375, 0.2281494140625, 1.501953125, -1.369140625, -0.2171630859375, 0.7392578125, -0.2052001953125, -0.296630859375, -0.8798828125, -1.19140625, -0.65185546875, -1.091796875, 1.0849609375, 0.264404296875, 0.3662109375, 1.8681640625, 1.1328125, 0.47607421875, -0.469970703125, -0.08575439453125, 0.6435546875, 0.56689453125, -0.939453125, -0.873046875, -1.5888671875, 0.49462890625, -0.2027587890625, 1.0400390625, 0.18115234375, 0.43603515625, 0.3623046875, 0.2294921875, 0.5302734375, -0.402587890625, 1.8896484375, 0.65380859375, -0.007076263427734375, 1.7314453125, -0.87548828125, -0.72265625, -0.401123046875, 0.7109375, 0.69873046875, -1.1064453125, 0.1951904296875, 0.326904296875, 1.8466796875, 0.28173828125, 0.703125, -0.45751953125, 0.40771484375, -0.48046875, 0.96044921875, 0.38525390625, -0.82080078125, 0.468994140625, -0.466064453125, -0.26513671875, 2.10546875, -0.86962890625, 0.8583984375, -1.0341796875, -0.64013671875, -0.357177734375, -0.2744140625, 1.39453125, -0.45361328125, -1.611328125, -0.66748046875, 0.98828125, 0.1895751953125, -2.0390625, 1.0625, -0.347900390625, 0.728515625, 1.0712890625, 1.09765625, 0.55908203125, 1.77734375, -0.60302734375, 0.118896484375, -0.17333984375, -1.173828125, -0.1572265625, -0.7451171875, -0.361328125, -1.201171875, -0.455810546875, 0.78662109375, 0.8544921875, 0.262939453125, -0.37744140625, 0.13037109375, -0.81884765625, -0.7900390625, 0.0188751220703125, 0.09979248046875, -0.97802734375, -0.039886474609375, -1.123046875, 1.1474609375, -1.720703125, 0.230712890625, 0.01335906982421875, 0.080078125, 1.869140625, -0.236328125, 1.8603515625, -2.84375, 0.124755859375, 0.440185546875, 0.282958984375, -0.2398681640625, 1.40625, 0.7041015625, -0.34619140625, -0.30078125, 2.62890625, 1.4677734375, -0.2401123046875, 0.50439453125, -1.3369140625, 0.01334381103515625, 0.85302734375, -0.68994140625, -0.0462646484375, 0.83642578125, 1.6044921875, 0.458984375, 0.91015625, 2.119140625, -0.048370361328125, -0.92578125, 0.66259765625, 0.72265625, 0.68017578125, 1.326171875, 0.30029296875, -1.1181640625, -0.278076171875, 0.8837890625, 1.0419921875, 1.0078125, 0.437744140625, -0.77392578125, 0.65234375, -1.1044921875, 0.85888671875, 0.0838623046875, 0.486083984375, -0.140380859375, -0.68701171875, 0.364990234375, 0.8466796875, -0.0650634765625, 0.053680419921875, 0.58544921875, 0.1387939453125, 0.2001953125, -0.1669921875, 0.37109375, 0.314453125, 0.9677734375, -0.0672607421875, 0.59375, -1.2626953125, 0.7861328125, -0.2388916015625, 0.92431640625, 0.9931640625, 0.96533203125, -0.408935546875, 0.8623046875, -0.0582275390625, -0.246337890625, -0.703125, 1.1875, -0.1590576171875, -0.1207275390625, -0.037445068359375, 0.41943359375, 0.08636474609375, -1.134765625, 0.1439208984375, -0.37646484375, -0.59912109375, -1.8623046875, 0.6640625, 0.2069091796875, -0.92138671875, -1.5087890625, 1.3173828125, -0.41259765625, -0.5869140625, -1.27734375, 0.1285400390625, -1.083984375, -1.107421875, 1.1435546875, 0.00696563720703125, 1.12109375, 0.78173828125, 1.697265625, 0.98095703125, 0.440185546875, -3.6640625, 0.46044921875, -1.7880859375, 0.65673828125, -0.74169921875, 0.708984375, -1.0927734375, 0.58642578125, 0.6162109375, -1.69140625, 0.11199951171875, -1.158203125, 0.5693359375, 0.78076171875, -1.7001953125, -0.5126953125, -0.305419921875, 0.044830322265625, 0.5830078125, -0.60107421875, -0.77734375, 0.0870361328125, -0.73779296875, -0.49853515625, 0.0706787109375, -0.04400634765625, -1.2353515625, 0.8623046875, 0.09625244140625, 1.5546875, -0.060699462890625, 0.787109375, 0.014495849609375, 1.3037109375, 1.6650390625, -1.6650390625, -0.107421875, 0.28662109375, -2.513671875, 0.57568359375, -0.8046875, 0.56787109375, -0.390869140625, 0.896484375, -2.443359375, 1.72265625, -0.77880859375, -1.1005859375, 0.151123046875, -0.8564453125, 0.4716796875, -0.703125, -0.37646484375, -1.021484375, 0.55712890625, 1.5126953125, 2.150390625, 0.7861328125, 0.3359375, -0.744140625, 0.51611328125, -0.7470703125, 1.0849609375, 0.292236328125, 1.44921875, 1.55859375, 0.7255859375, 1.06640625, -0.220458984375, 2.140625, 0.446044921875, 2.404296875, -0.5078125, -0.59326171875, -0.70458984375, -2.865234375, 1.603515625, -0.5458984375, 0.6318359375, -2.94140625, -0.787109375, -0.142333984375, -0.28125, -0.62548828125, -1.17578125, -0.9541015625, 0.24658203125, -0.385986328125, -0.619140625, 0.3740234375, -1.5009765625, -0.41357421875, -0.137451171875, -0.5126953125, 1.501953125, -0.77685546875, 0.36083984375, 1.181640625, -0.33056640625, 0.623046875, 0.759765625, -0.74755859375, 0.28369140625, -0.0882568359375, 0.97265625, -0.01036834716796875, -1.07421875, 0.244873046875, -0.403076171875, 0.43408203125, -0.95263671875, -0.390380859375, 0.489990234375, -0.385986328125, 0.552734375, 0.8662109375, -0.853515625, 0.78955078125, -0.8662109375, 1.533203125, 0.59814453125, 0.55859375, 0.67138671875, -0.463623046875, 0.88330078125, -0.64208984375, 0.3818359375, 1.2099609375, 0.06719970703125, 0.12548828125, 1.142578125, -1.3212890625, -0.446533203125, 0.759765625, -0.090087890625, 1.6201171875, 0.386474609375, -0.97607421875, -0.8515625, 0.406005859375, -0.556640625, -0.03851318359375, -0.0989990234375, -1.5537109375, -1.3984375, -1.466796875, -2.478515625, 0.87255859375, -0.40771484375, 2.912109375, 0.31396484375, 0.5927734375, 2.478515625, 1.1025390625, -0.77587890625, -0.13525390625, 0.1251220703125, 0.103271484375, -0.269775390625, 0.403564453125, 0.21923828125, -1.75, -0.9990234375, 1.1103515625, -0.8916015625, -0.7939453125, 0.80078125, 1.8427734375, 1.248046875, -2.240234375, 0.283203125, -0.1533203125, 0.85107421875, 0.49853515625, 0.671875, 1.4951171875, 1.279296875, -0.97607421875, 0.6728515625, 0.385009765625, 0.56396484375, -0.18017578125, 0.011077880859375, 1.1826171875, -0.2025146484375, -0.396240234375, 0.421142578125, -0.68505859375, 0.88623046875, 2.033203125, -0.56640625, -0.54931640625, -0.5634765625, -1.177734375, 0.2408447265625, 1.34765625, -1.478515625, 1.1953125, 0.5908203125, -0.5234375, -0.0120391845703125, 0.60498046875, 0.08502197265625, 0.032318115234375, -0.3271484375, -0.2154541015625, -0.13232421875, -0.67333984375, 0.91552734375, 0.46533203125, 0.75390625, 0.0177764892578125, 0.08837890625, 0.25244140625, 0.386962890625, 0.47509765625, -1.23828125, -0.76708984375, 0.796875, -0.2239990234375, -0.1103515625, -0.98828125, 1.91015625, 2.04296875, -0.8701171875, 0.04754638671875, -1.333984375, 1.26953125, -0.29443359375, 0.68212890625, 0.587890625, -0.48681640625, -0.92626953125, 0.25048828125, -0.7021484375, -0.006134033203125, 0.7265625, -0.32470703125, -0.97314453125, 0.3408203125, 0.6259765625, -0.268798828125, 0.50439453125, -1.265625, 0.204345703125, 0.24755859375, -0.2076416015625, 1.5537109375, 0.188232421875, 0.7822265625, -0.10400390625, -0.46435546875, 1.8720703125, -0.62548828125, 0.98779296875, -0.23779296875, 1.3623046875, 0.1209716796875, 0.10711669921875, -0.6162109375, -0.50830078125, -0.1407470703125, 1.4013671875, -0.578125, 0.382080078125, 0.72216796875, -0.06451416015625, 1.1484375, -0.6796875, 0.055145263671875, -0.460205078125, -1.412109375, -0.60107421875, -0.141357421875, -1.291015625, 1.12890625, 0.5380859375, 0.92724609375, 0.06646728515625, -0.49951171875, 0.2230224609375, -0.59423828125, 0.12176513671875, 0.1728515625, -0.58544921875, -1.009765625, 0.2183837890625, -0.464111328125, 0.56494140625, 0.1517333984375, -0.2008056640625, -1.2978515625, -1.6943359375, 1.0625, -0.165283203125, -2.765625, 0.533203125, -0.312255859375, -0.2283935546875, -0.92236328125, -1.7373046875, -0.46728515625, -0.541015625, 1.568359375, -0.319580078125, 1.078125, -0.5703125, -0.466796875, -1.2900390625, -1.58203125, -0.2568359375, -0.153076171875, 0.317138671875, -0.544921875, 0.72314453125, -1.4970703125, -0.174072265625, 0.5947265625, -0.685546875, -0.1302490234375, 0.08636474609375, -0.294189453125, 0.9111328125, 0.2347412109375, 1.3115234375, -0.76318359375, -0.87646484375, -0.5205078125, 0.69775390625, -0.343017578125, -0.1719970703125, 0.09515380859375, 1.1533203125, 0.615234375, -0.39990234375, 2.232421875, -0.76220703125, -1.9443359375, -0.482177734375, 0.5830078125, -0.234130859375, 1.68359375, 0.85986328125, -0.578125, 0.3408203125, -0.363525390625, 0.966796875, -1.0546875, -0.65283203125, 1.2197265625, -0.38818359375, 0.27099609375, 1.0244140625, -1.5224609375, -0.7705078125, -0.39208984375, 1.2294921875, -0.438720703125, 0.437255859375, 1.5322265625, -0.9462890625, -0.83642578125, -0.082763671875, -0.99951171875, -0.297607421875, 0.44091796875, 0.1773681640625, -0.662109375, -0.319091796875, 0.0235137939453125, -0.16064453125, 0.414794921875, -0.1341552734375, -0.82421875, -0.0703125, -1.1201171875, -0.139892578125, -1.6962890625, -0.07269287109375, 0.7109375, -1.0087890625, -0.018341064453125, -0.6669921875, -0.218994140625, -1.0439453125, 1.337890625, -1.0107421875, -1.5439453125, 2.00390625, -0.25244140625, -1.080078125, -0.814453125, -0.572265625, -0.3193359375, 0.1923828125, 0.47900390625, -0.546875, -0.0994873046875, -0.0271148681640625, 0.092529296875, -0.8525390625, -1.017578125, -0.5732421875, -0.27978515625, 0.0360107421875, -1.53515625, 0.1202392578125, -1.62890625, 1.4228515625, -0.59716796875, -0.126220703125, -2.123046875, -1.1806640625, -0.1947021484375, 1.5, 2.08203125, 0.71142578125, 0.93798828125, -0.0833740234375, 0.076904296875, -0.7900390625, 1.06640625, 0.376708984375, -0.044219970703125, -0.57080078125, 0.87353515625, -1.482421875, 1.7587890625, 0.3681640625, 2.064453125, -1.0556640625, -0.01032257080078125, 1.79296875, 0.2174072265625, -0.955078125, -0.6552734375, 0.314453125, -0.426513671875, 0.1578369140625, 1.08984375, -0.1759033203125, -0.8076171875, 0.08038330078125, -1.1220703125, 1.1279296875, 0.41748046875, -1.5302734375, 0.189697265625, 0.67578125, 1.2314453125, -1.142578125, 1.6357421875, 0.070556640625, 2.76953125, 0.1949462890625, -2.28515625, 0.41796875, 0.54931640625, 0.572265625, 1.31640625, 0.90771484375, -0.6064453125, 0.41845703125, -1.3193359375, 0.143798828125, 0.27685546875, 1.4169921875, -0.67431640625, 0.76171875, -0.260498046875, -0.7333984375, -1.3779296875, 0.03326416015625, 1.12109375, 0.5625, 0.564453125, 0.3857421875, 1.9658203125, 1.1123046875, 0.2432861328125, 0.65478515625, -0.4970703125, -0.0438232421875, -0.2117919921875, -1.62109375, 0.57958984375, 1.76953125, 0.953125, 0.404296875, -1.1787109375, 1.8857421875, 1.509765625, -1.529296875, -0.0750732421875, 0.69970703125, -0.7451171875, -0.0013217926025390625, 0.435791015625, 0.445556640625, -0.38330078125, 0.389404296875, 0.7568359375, 0.421875, -0.1572265625, -0.167724609375, 0.75537109375, 2.009765625, -1.3291015625, 0.348876953125, 0.01175689697265625, -0.45751953125, 0.0027008056640625, 0.0751953125, -0.775390625, 1.4404296875, -0.053741455078125, 0.1761474609375, -0.484619140625, 0.0125885009765625, -0.96728515625, 0.10540771484375, 0.2467041015625, -1.83203125, 0.94921875, 0.38134765625, -0.444091796875, 1.384765625, -0.2734375, 0.8203125, -1.8115234375, -1.1162109375, -1.193359375, -0.168701171875, -0.2724609375, 0.55615234375, 0.287841796875, 0.63720703125, 0.5537109375, 1.6533203125, 0.53125, 1.8427734375, -2.93359375, 1.8603515625, 1.666015625, -1.138671875, -2.3203125, -0.52490234375, -0.77978515625, 0.31591796875, 0.59326171875, 2.591796875, 0.84716796875, -0.6171875, 0.66015625, -0.75927734375, 0.1998291015625, -0.01715087890625, -0.1492919921875, 0.67626953125, -1.1513671875, -0.352294921875, -0.1805419921875, -1.6982421875, -1.1318359375, -0.06536865234375, -0.12237548828125, -0.377197265625, -0.57177734375, -1.0234375, -0.6572265625, -0.72119140625, 0.433837890625, 0.425048828125, -0.357177734375, -0.10205078125, 0.54833984375, 0.08819580078125, 0.78271484375, -0.2283935546875, 0.3427734375, 0.48046875, -0.9580078125, -2.353515625, -0.1558837890625, 0.6201171875, 1.931640625, -1.0986328125, -1.8720703125, -0.08892822265625, 1.177734375, 0.5048828125, -1.8134765625, 1.2509765625, -0.395263671875, -1.15234375, 0.4443359375, -1.681640625, 0.12078857421875, -0.62939453125, 0.11029052734375, 0.024658203125, -1.1845703125, 0.7724609375, 0.5888671875, 0.488037109375, 1.430511474609375e-06, -0.8173828125, 0.99755859375, 0.4599609375, -1.2529296875, 0.266357421875, 0.1436767578125, 0.63525390625, -0.8974609375, 0.515625, 0.050048828125, 1.1455078125, 0.87158203125, 0.474609375, 1.1123046875, -1.96875, 0.568359375, 0.73486328125, -0.28271484375, 0.73779296875, -0.09814453125, 1.7373046875, 1.6376953125, 1.0283203125, -3.083984375, -0.377197265625, 0.18310546875, 0.77392578125, -0.94580078125, 0.06842041015625, -0.207275390625, -0.25390625, 1.0224609375, -2.119140625, -1.62109375, -0.371337890625, 1.197265625, -0.7783203125, -0.10711669921875, 0.90966796875, -0.3515625, 0.48681640625, -0.720703125, -0.325439453125, 0.62890625, 1.1552734375, -0.2340087890625, 0.58642578125, 2.4140625, -0.6376953125, -0.441162109375, 0.94189453125, 1.5, -0.8994140625, 1.439453125, 0.1461181640625, 0.830078125, 0.8017578125, 0.5146484375, 0.4873046875, 0.27197265625, -0.71826171875, -0.59130859375, 0.97412109375, -0.59130859375, 0.44873046875, -1.1923828125, 0.841796875, -0.320068359375, -0.42919921875, -0.2178955078125, -0.80712890625, 0.1455078125, -0.67431640625, -1.1142578125, -0.55126953125, -0.0211639404296875, 0.6142578125, -1.4208984375, -1.0009765625, -0.79150390625, 1.2236328125, 0.0421142578125, -0.488525390625, -0.56787109375, 1.3701171875, 0.60791015625, -0.37841796875, -0.49853515625, -0.1519775390625, 1.1416015625, -1.4541015625, 0.67822265625, 0.423095703125, 0.50439453125, -0.423583984375, 0.24462890625, 0.08807373046875, -0.342529296875, 0.615234375, 0.568359375, 0.07879638671875, -1.080078125, -2.357421875, -0.11920166015625, 0.181396484375, -1.634765625, 1.1103515625, -0.038787841796875, 0.41650390625, 0.03045654296875, -1.017578125, 0.2919921875, -0.1737060546875, -0.085693359375, -1.4951171875, -0.1114501953125, -1.5966796875, 0.278564453125, 0.0994873046875, -0.5009765625, 0.7685546875, -0.81982421875, -1.5537109375, 0.84033203125, 2.28125, 0.97509765625, -1.05859375, -0.099609375, 0.380859375, -0.9599609375, -0.53076171875, -0.283447265625, -0.5146484375, 0.05267333984375, -0.55517578125, 0.78173828125, 0.51220703125, 1.7724609375, -0.9287109375, 0.320068359375, 0.9833984375, 0.92919921875, -2.109375, -0.59375, 1.38671875, -0.7861328125, -0.30126953125, -1.2900390625, 0.7392578125, 1.4150390625, 0.274169921875, 0.93017578125, 0.0199737548828125, -0.29833984375, -0.79638671875, 0.8173828125, -0.222900390625, 2.59765625, -0.452880859375, -1.404296875, -0.69775390625, -0.57568359375, 1.4609375, 0.084228515625, -0.08624267578125, 1.3330078125, -1.609375, -2.578125, 0.337646484375, 0.0289154052734375, -2.859375, 0.61962890625, 0.359375, -0.87841796875, -0.61474609375, 0.76953125, 0.8134765625, -0.186767578125, 0.5947265625, 0.5390625, 1.3798828125, 0.36572265625, -0.33544921875, 0.52392578125, -0.200927734375, -0.58837890625, -1.916015625, 0.445556640625, -0.389892578125, 2.765625, 1.017578125, 0.83935546875, -2.099609375, 1.5810546875, 0.43115234375, -1.8740234375, 0.2064208984375, -0.873046875, -0.90771484375, -1.1123046875, -0.1295166015625, -1.810546875, 0.986328125, 1.1171875, 0.1553955078125, -0.1458740234375, -2.328125, 0.615234375, 0.485595703125, 0.72021484375, -1.75390625, 0.58203125, -0.378173828125, 0.96484375, 2.171875, 0.248291015625, -0.18212890625, -0.1009521484375, -1.06640625, -0.72802734375, -0.1455078125, -0.1263427734375, 0.89111328125, -1.240234375, 1.130859375, 0.1407470703125, 1.0322265625, 0.818359375, -0.64208984375, 1.130859375, -1.3349609375, 0.77099609375, -0.513671875, 1.2119140625, -0.9208984375, -0.73486328125, -0.6572265625, 0.447265625, -0.900390625, -0.53955078125, 0.07257080078125, 0.3388671875, -0.4453125, -0.72802734375, 0.2802734375, 0.2393798828125, 0.73828125, 1.00390625, 1.546875, 0.45556640625, -0.90771484375, 0.2362060546875, 0.09173583984375, -0.155517578125, 2.041015625, 0.7705078125, -0.5615234375, -0.0606689453125, 0.681640625, -2.5234375, -0.595703125, -1.7294921875, 0.2021484375, 0.390625, 1.083984375, 0.0916748046875, -0.99072265625, 0.267822265625, 1.8203125, 0.95458984375, -0.038177490234375, -0.30224609375, 0.09344482421875, -0.447998046875, -0.32568359375, 0.1331787109375, -0.2403564453125, 0.04864501953125, 1.103515625, -1.658203125, -0.53369140625, 0.3515625, -0.26416015625, -0.6181640625, 0.07745361328125, -0.55078125, -1.73046875, -1.5126953125, 2.087890625, -0.1326904296875, 0.53173828125, -0.53173828125, 1.3466796875, -0.461181640625, 0.1846923828125, -0.63623046875, -0.057891845703125, 0.419189453125, 0.693359375, -0.3349609375, -1.0517578125, 1.6298828125, -0.1524658203125, -1.65625, 1.0341796875, -0.1407470703125, 0.07379150390625, -1.0400390625, 0.1192626953125, -0.81787109375, -0.08203125, -0.5693359375, 0.2822265625, -0.092529296875, -0.433349609375, -0.0249176025390625, -0.472412109375, -0.98291015625, -1.013671875, -0.990234375, 0.39404296875, 0.92529296875, 1.349609375, 0.3701171875, 0.4013671875, 1.162109375, 0.419921875, 0.1268310546875, 0.5849609375, 0.339599609375, 0.90869140625, 1.2626953125, 0.4013671875, 0.14599609375, 0.4609375, -2.787109375, -2.13671875, -0.89453125, 0.348876953125, -0.5439453125, -0.214111328125, -1.5068359375, -0.2425537109375, -0.51904296875, -0.25341796875, 1.0732421875, -0.63037109375, -0.6708984375, -0.162109375, -0.2073974609375, -1.2265625, 0.21240234375, 0.296142578125, -1.244140625, 0.391357421875, 0.1260986328125, -0.00959014892578125, 0.16064453125, -0.2734375, 1.0595703125, -0.375244140625, -0.1531982421875, 0.26123046875, -0.439208984375, -0.0540771484375, 0.177734375, 0.288818359375, -0.93896484375, 0.2626953125, 0.892578125, -1.0693359375, 1.6015625, 1.392578125, 0.86181640625, 1.306640625, -0.29541015625, 0.26611328125, -0.81884765625, -0.383544921875, 0.84619140625, -0.00806427001953125, 0.9990234375, -0.494873046875, 0.40625, -1.0673828125, -0.291259765625, 0.6357421875, -0.89013671875, 0.55517578125, 0.2254638671875, -1.1982421875, 0.68701171875, -0.46337890625, 0.134521484375, -0.106689453125, -3.26953125, -0.0555419921875, 0.03558349609375, 0.6123046875, -0.438232421875, -0.55029296875, -0.27001953125, -0.477783203125, -0.4658203125, 0.841796875, 1.9296875, -0.7314453125, 1.0234375, 1.4990234375, -0.2890625, -0.08868408203125, -1.1767578125, 0.6533203125, -1.09375, -0.9384765625, -0.8544921875, 0.9775390625, 0.658203125, 0.4912109375, -0.1678466796875, -0.39599609375, 1.1435546875, 0.340087890625, 0.73291015625, 0.8115234375, 0.261962890625, -0.07574462890625, 1.3681640625, 1.193359375, 0.031463623046875, 0.13623046875, -0.482421875, 0.25, -1.38671875, -0.9931640625, 0.332275390625, -0.233642578125, -1.080078125, 0.12744140625, -0.86376953125, 0.262939453125, -0.306640625, -0.10260009765625, 1.0576171875, -1.0009765625, -1.0283203125, -0.66748046875, 0.002086639404296875, -0.06512451171875, -0.045379638671875, 0.10723876953125, 1.1650390625, -0.68603515625, 1.0068359375, -0.56982421875, 2.19140625, -0.253173828125, -1.8623046875, -0.1741943359375, -2.49609375, -2.23828125, 1.0478515625, 0.6474609375, 1.501953125, 0.261474609375, -0.8330078125, -0.195068359375, 1.029296875, 0.52685546875, -1.66796875, 0.58349609375, -0.454345703125, 2.12890625, -0.6982421875, -0.4326171875, 2.30859375, 2.224609375, -1.1650390625, -1.3388671875, 0.5380859375, 1.18359375, 0.23779296875, -0.1319580078125, 0.400634765625, 0.65966796875, -0.81591796875, -0.89892578125, -0.31298828125, 1.2412109375, -1.17578125, 2.05078125, -1.41796875, -0.258544921875, -0.56787109375, -1.19921875, 0.71826171875, 0.720703125, 1.4228515625, 1.4697265625, -0.578125, -0.8349609375, -0.291259765625, 0.80078125, 0.904296875, -0.6796875, 0.055419921875, -0.08349609375, 1.052734375, 1.1064453125, -0.323486328125, -0.71435546875, -0.3271484375, 0.99951171875, 1.099609375, 0.78955078125, 1.689453125, -2.640625, 1.240234375, 1.296875, -0.5576171875, 0.2152099609375, 0.94580078125, -0.0141143798828125, -1.6083984375, 1.744140625, -0.129638671875, 1.265625, 0.62353515625, -0.76904296875, 0.446533203125, 0.332763671875, 0.7021484375, 0.19775390625, -0.53173828125, 1.11328125, 0.487548828125, -1.5732421875, -0.37353515625, -2.16015625, -0.43701171875, 0.708984375, -0.350830078125, 0.75048828125, 0.38525390625, 0.283447265625, 0.82568359375, 2.224609375, 1.4453125, -0.55615234375, -1.646484375, 0.1815185546875, 0.94287109375, -0.44970703125, 0.7353515625, 0.54345703125, -0.7744140625, 0.114990234375, 0.1702880859375, -0.888671875, -0.0015687942504882812, 0.225830078125, -1.39453125, 1.232421875, -1.4404296875, -0.69873046875, 0.265869140625, -0.32080078125, 0.97412109375, -0.151611328125, 1.833984375, -0.17529296875, -1.154296875, 2.064453125, -1.9384765625, 0.76708984375, 1.279296875, -1.0068359375, 1.1943359375, 1.671875, 0.80078125, 0.28564453125, 1.033203125, 0.7197265625, 0.90234375, 2.01953125, 1.2783203125, -2.177734375, -1.2734375, 0.03851318359375, -0.157470703125, -0.1376953125, 1.1904296875, 0.640625, -0.53125, 1.1484375, 0.42724609375, 0.64453125, 0.60498046875, -0.41357421875, 0.6064453125, 0.397705078125, 0.12060546875, 1.0029296875, -0.744140625, 1.0615234375, 1.6474609375, 0.70947265625, 0.0892333984375, -1.0791015625, -1.0869140625, -0.291748046875, 0.7734375, -1.28515625, -0.1795654296875, 0.9150390625, -0.67236328125, -0.4765625, -1.3193359375, 0.9931640625, 0.98828125, 0.884765625, 1.474609375, 0.6201171875, -1.7841796875, 1.5751953125, 0.2235107421875, 2.376953125, 1.1318359375, 0.10272216796875, -0.1103515625, 1.0185546875, 1.486328125, 0.71630859375, -0.47265625, -1.3828125, -0.9091796875, -1.1552734375, -1.31640625, 0.02020263671875, 0.5849609375, -0.00907135009765625, -2.6484375, -1.1181640625, -0.55419921875, -2.3046875, 0.393310546875, -1.5283203125, -1.03515625, 0.2232666015625, -0.1220703125, 0.09759521484375, -0.71240234375, 0.335693359375, 0.57080078125, -0.001590728759765625, 0.444580078125, 0.85009765625, -0.44287109375, -0.0308074951171875, -1.1162109375, -0.951171875, 0.74755859375, 1.01953125, -0.61865234375, -0.61669921875, 0.480224609375, -0.75927734375, 1.9326171875, -1.724609375, -0.5234375, 0.71630859375, 0.03607177734375, -1.0283203125, 1.0849609375, -0.810546875, -0.1986083984375, 1.24609375, -0.69482421875, 0.62939453125, -0.1334228515625, -0.46826171875, -0.69091796875, -0.0789794921875, 1.1298828125, -0.206298828125, -1.154296875, -0.23779296875, -0.50634765625, -0.59521484375, -0.685546875, -0.328369140625, 0.36279296875, -0.90576171875, -0.88916015625, -0.53466796875, -1.171875, -0.2490234375, 1.5009765625, 1.7509765625, -0.1328125, 0.306640625, 0.11309814453125, 1.25, -0.27392578125, -1.568359375, -0.025390625, 0.318115234375, 0.029998779296875, 0.5908203125, -1.865234375, -1.3779296875, -0.1395263671875, 0.2386474609375, -0.81884765625, -0.697265625, -0.8857421875, -0.2476806640625, -0.44140625, -0.24267578125, -0.1756591796875, -1.3837890625, 0.57421875, -0.362548828125, 1.0380859375, 0.1395263671875, 1.91015625, -0.50537109375, 0.09564208984375, -0.8984375, -1.6142578125, 0.0953369140625, -1.3984375, -3.271484375, -0.91748046875, -1.0224609375, -1.453125, -0.3388671875, 0.1142578125, -1.23046875, 1.3515625, 2.119140625, -0.3427734375, -0.44189453125, -0.300048828125, -0.2890625, -0.280517578125, -1.2158203125, 1.2548828125, 0.9755859375, 0.136474609375, 0.05889892578125, -0.8232421875, 0.98046875, -0.80712890625, 0.419921875, -1.005859375, -0.30419921875, -0.1884765625, -0.07244873046875, -0.95751953125, -0.497314453125, -0.1968994140625, 0.2266845703125, -0.2122802734375, 0.94970703125, -0.26708984375, 0.10504150390625, 1.1220703125, 0.3125, 2.15234375, -0.3720703125, 0.89794921875, 0.82861328125, -0.459228515625, 0.96142578125, 1.306640625, 1.2529296875, -0.41748046875, -0.8466796875, -0.146484375, -0.0010137557983398438, 0.583984375, -0.52587890625, -0.0216522216796875, 0.050201416015625, -0.38427734375, 0.572265625, 0.693359375, -1.0146484375, 0.57373046875, 0.0955810546875, 0.05670166015625, -0.102294921875, -2.056640625, -1.19921875, 0.6982421875, -0.904296875, -0.94482421875, -0.1475830078125, 0.76708984375, 0.42822265625, -1.0546875, 0.5400390625, 1.0126953125, -1.3212890625, -0.78466796875, 0.233642578125, 1.05078125, -0.79931640625, 0.1904296875, 0.3671875, -0.458984375, 0.7734375, 0.177490234375, -0.8603515625, 0.1221923828125, -0.64599609375, -0.187255859375, -1.0859375, -0.260498046875, 0.2763671875, 0.474609375, -2.96875, 0.2418212890625, -0.50146484375, -0.72900390625, -1.158203125, 1.677734375, -0.96142578125, -1.349609375, 1.373046875, 1.615234375, -0.63623046875, -2.052734375, -0.354736328125, 0.262451171875, 0.7666015625, -0.68310546875, 1.2197265625, -0.82763671875, 1.2451171875, 0.0458984375, 0.95556640625, 1.3134765625, 0.2210693359375, 0.132568359375, -0.39892578125, 0.5673828125, 0.953125, -0.279296875, 0.154052734375, -0.126220703125, 1.8740234375, 2.7578125, -0.135498046875, 0.272705078125, -0.70654296875, -0.35888671875, -1.41015625, -0.409912109375, 0.53564453125, -0.329833984375, 0.1553955078125, -1.1875, -0.55126953125, -0.6708984375, 0.5634765625, -0.71923828125, -0.339111328125, 0.9873046875, -2.11328125, 1.677734375, 0.313720703125, 0.357666015625, 0.434814453125, 0.04937744140625, -0.607421875, 0.25048828125, -0.8310546875, 0.3935546875, 0.84375, -0.5009765625, -0.425048828125, -0.201416015625, 1.3291015625, 1.142578125, -0.4501953125, 0.457275390625, 0.01264190673828125, 0.62109375, -0.4501953125, 0.21630859375, -0.96435546875, -0.10870361328125, -0.1856689453125, 1.0380859375, -0.564453125, 0.5810546875, -0.1256103515625, 0.38525390625, -0.247802734375, -0.734375, 0.30712890625, -1.2041015625, -0.1998291015625, 0.53857421875, -0.1925048828125, 2.537109375, -1.5693359375, -0.705078125, -0.7919921875, -1.734375, 1.345703125, 0.047637939453125, -1.939453125, 0.11358642578125, 0.07415771484375, 0.11541748046875, 1.0546875, -0.2193603515625, 0.364990234375, -0.2548828125, -0.09344482421875, -0.583984375, -1.8310546875, -2.314453125, -0.1959228515625, -0.529296875, -1.322265625, 0.69189453125, -0.26953125, -0.385986328125, -1.3427734375, 1.8828125, 0.66845703125, 0.0709228515625, -0.32373046875, 1.2578125, -0.278076171875, -0.01448822021484375, -0.1475830078125, -0.040802001953125, 0.255126953125, -1.19140625, 0.54638671875, 1.2158203125, 0.62646484375, 0.10552978515625, 0.357666015625, 0.472900390625, 0.1566162109375, -0.0494384765625, 1.705078125, 0.70458984375, 0.60107421875, 0.09161376953125, 0.48828125, 0.358642578125, -1.712890625, -0.51220703125, -0.2244873046875, -0.07635498046875, 1.486328125, -2.931640625, -0.84765625, 0.12322998046875, -0.040130615234375, -0.399169921875, -0.70361328125, -1.1494140625, -0.8251953125, 0.60791015625, 0.65380859375, -1.1181640625, -0.59521484375, -1.537109375, 0.7998046875, -1.748046875, 0.40185546875, 0.1380615234375, 0.61865234375, -1.0595703125, 0.49560546875, 1.8173828125, 0.4794921875, -0.0256195068359375, -0.88232421875, 2.04296875, 0.202880859375, -1.7001953125, 0.261962890625, 0.32861328125, 0.791015625, -0.07183837890625, -0.5390625, 1.3134765625, 0.8759765625, -1.126953125, 1.5947265625, -1.8212890625, 1.65625, 0.1234130859375, -1.0263671875, -0.354248046875, -0.60107421875, 1.3076171875, -0.334228515625, 0.041748046875, -0.0306396484375, 0.24853515625, -1.65234375, 0.91064453125, -1.068359375, -0.50634765625, -0.351806640625, 1.0400390625, 0.6298828125, 0.56640625, 1.8701171875, -0.81298828125, 0.369140625, 0.84619140625, 0.364501953125, -2.03125, 0.35595703125, 1.7314453125, -0.7431640625, 0.5390625, 0.97119140625, -0.08880615234375, 1.455078125, -2.771484375, 2.38671875, -0.6337890625, 1.90234375, -0.39111328125, -0.04449462890625, -0.74951171875, -1.900390625, 1.0947265625, 0.1485595703125, 0.884765625, -0.3134765625, -0.79296875, 0.01043701171875, -0.397705078125, -1.162109375, 1.390625, 0.98095703125, -0.338134765625, -1.5126953125, -1.16796875, 0.51416015625, 0.66162109375, -1.708984375, -0.19384765625, 0.658203125, -0.297607421875, -0.251708984375, 0.43408203125, -0.10467529296875, 0.60888671875, -0.85693359375, 1.4228515625, -0.7919921875, 1.5107421875, 1.294921875, 1.42578125, 0.57275390625, 1.6396484375, 0.73193359375, 1.0712890625, 0.0098419189453125, -0.83251953125, -0.1837158203125, -1.4599609375, 0.1531982421875, 0.82861328125, 1.2333984375, 0.326416015625, -0.861328125, -0.66162109375, -1.4716796875, 1.193359375, -1.0244140625, -0.061767578125, 0.67919921875, -1.1044921875, -0.8349609375, 1.0810546875, -1.5625, 0.25341796875, -0.37255859375, -0.07208251953125, -0.91455078125, -0.365234375, 0.77978515625, 1.1611328125, -1.099609375, 2.470703125, -0.4951171875, 0.6689453125, -0.572265625, -0.59814453125, 1.1591796875, 0.166015625, 0.043243408203125, 1.5546875, -0.97998046875, 0.32177734375, -0.27099609375, 1.125, -0.33251953125, -1.6796875, 1.6455078125, 0.52978515625, -0.398681640625, 0.64599609375, 0.96923828125, 0.7236328125, -1.248046875, 0.7978515625, -0.404296875, -0.12396240234375, 0.2216796875, -1.38671875, -1.5322265625, -0.90087890625, 0.058258056640625, 1.517578125, -0.44189453125, -0.491455078125, 0.95654296875, 0.33349609375, -0.89794921875, 0.149658203125, -0.61474609375, 1.044921875, -0.494140625, 0.52880859375, 0.38818359375, -1.0341796875, -0.257568359375, 0.158935546875, -0.7177734375, 0.3505859375, 0.31298828125, -1.3017578125, 0.55029296875, -0.01013946533203125, 0.097900390625, 0.15283203125, -0.425537109375, -0.39892578125, 0.8251953125, 0.5126953125, 0.059967041015625, -0.7724609375, 0.023040771484375, -1.642578125, -0.70751953125, 0.2281494140625, -1.091796875, 0.382080078125, 0.72802734375, -1.0869140625, -0.73046875, 1.185546875, -0.1434326171875, -0.7421875, 0.41650390625, 0.264404296875, -0.8828125, 1.09765625, 0.2137451171875, 0.43994140625, -0.1331787109375, 0.050872802734375, -0.02496337890625, -0.59814453125, -0.450927734375, -0.55419921875, 0.21875, -0.7607421875, 0.21923828125, -0.2213134765625, 0.31689453125, -0.2880859375, -0.26513671875, -0.75927734375, 0.5, 0.48876953125, -0.0012464523315429688, -0.2265625, 1.234375, 0.44140625, 1.15234375, 0.74658203125, -0.06439208984375, 0.77880859375, 1.8779296875, -0.48193359375, 0.630859375, -1.0078125, 1.189453125, -2.560546875, -0.1700439453125, -1.201171875, 0.45556640625, 0.30859375, 0.264404296875, 0.8583984375, 0.0185699462890625, -0.39111328125, 0.8232421875, -1.4130859375, 0.477294921875, -0.1353759765625, -1.564453125, -2.357421875, -0.290771484375, 0.97900390625, -0.5693359375, 0.6572265625, 1.111328125, 0.188232421875, -1.796875, 0.41015625, 0.1588134765625, 0.67138671875, -0.67529296875, 0.408935546875, -0.86328125, -1.7666015625, 0.456787109375, 1.1865234375, -0.485107421875, -0.62646484375, 0.564453125, -1.0751953125, -0.857421875, -3.017578125, 0.1468505859375, 0.09002685546875, -0.81494140625, -0.89404296875, -0.68505859375, -0.9140625, 0.990234375, 0.03717041015625, 0.66015625, 0.017120361328125, -0.95068359375, -0.220458984375, 0.9033203125, -0.366455078125, -0.95458984375, 1.1494140625, -0.7421875, -0.011260986328125, 0.6533203125, 0.3583984375, -0.36474609375, -1.0283203125, -0.73095703125, -0.342041015625, -0.349365234375, -1.65625, 0.6806640625, -1.0458984375, 0.244384765625, -0.466552734375, -0.75341796875, 1.3896484375, -0.378662109375, -0.8134765625, -0.0089263916015625, 0.931640625, 0.75439453125, 0.09039306640625, 0.99365234375, 1.3447265625, -1.201171875, -0.1568603515625, 0.442626953125, -0.01190948486328125, -0.83349609375, -0.434814453125, -0.64501953125, -0.0157928466796875, -0.0675048828125, -1.1005859375, 2.22265625, -1.251953125, -0.2274169921875, -2.06640625, 0.08380126953125, 0.5361328125, -1.09375, 0.408447265625, 1.48046875, 0.35498046875, 0.97119140625, -0.64404296875, 0.0228729248046875, 1.5830078125, -1.0732421875, 1.1435546875, -0.91650390625, 1.896484375, 0.8525390625, 0.2110595703125, 0.08343505859375, -0.09600830078125, -1.15625, 0.01296234130859375, -0.68310546875, 0.00968170166015625, 0.97802734375, 0.67431640625, -0.33056640625, 1.0, -0.10272216796875, -0.1749267578125, 1.6103515625, -0.1011962890625, -1.091796875, 0.353271484375, 0.4423828125, -1.658203125, -1.150390625, -1.3369140625, 0.51025390625, -1.5244140625, 0.377197265625, -2.509765625, -1.1455078125, 0.79736328125, 0.69677734375, -1.59375, -0.2490234375, -2.267578125, 0.334716796875, -0.75146484375, -1.76953125, -0.77392578125, 1.4052734375, 0.6513671875, 0.82373046875, 1.0986328125, -1.7021484375, 0.83642578125, 0.1534423828125, 0.00567626953125, 0.708984375, 0.17822265625, -0.2374267578125, 1.1005859375, -0.66796875, -0.1610107421875, -1.1435546875, 0.26806640625, -0.01319122314453125, 0.11572265625, 0.387451171875, 1.7919921875, 0.480712890625, 0.40869140625, 0.195068359375, -0.64990234375, -0.00630950927734375, 0.564453125, -1.6123046875, -0.7265625, -0.110107421875, -0.34716796875, -0.80419921875, 0.239013671875, 0.78662109375, 0.7080078125, -0.66455078125, 2.408203125, -0.31005859375, -0.306884765625, 1.4658203125, -0.72998046875, 0.0197296142578125, -1.65234375, 0.62744140625, -1.181640625, -1.62890625, -0.703125, -1.90625, -0.556640625, 0.1593017578125, -0.04547119140625, -0.148681640625, -1.4345703125, -0.78515625, 0.5869140625, -0.5888671875, -0.284423828125, 1.4638671875, -0.7666015625, 0.41748046875, -1.0380859375, -0.26220703125, 0.11785888671875, -1.3671875, -0.8955078125, -0.5517578125, 1.130859375, -0.07489013671875, -0.05499267578125, 0.54833984375, 0.7939453125, 0.51123046875, -0.2025146484375, 0.77001953125, 2.064453125, 0.78759765625, -0.171630859375, 0.66162109375, 0.407470703125, -0.79248046875, -1.521484375, 0.7509765625, 0.26025390625, 0.443115234375, -0.435791015625, 0.51611328125, -1.091796875, -0.52001953125, -0.398681640625, -1.4697265625, -0.463134765625, -0.04644775390625, -0.1793212890625, -1.34375, 2.359375, 2.345703125, 1.1005859375, -0.208740234375, -1.330078125, 0.1973876953125, -0.04473876953125, -1.328125, 0.430908203125, -1.0126953125, 1.2041015625, 0.82958984375, -0.892578125, -0.8994140625, 0.2200927734375, 2.0390625, -0.658203125, 0.73779296875, 2.501953125, 0.61669921875, -0.53125, -1.171875, -1.189453125, -0.1343994140625, 1.7421875, -0.283203125, 0.24462890625, 0.04254150390625, 0.0258941650390625, 0.0765380859375, -0.323486328125, 0.7939453125, -0.76806640625, -0.99609375, 1.255859375, -0.47705078125, 0.00611114501953125, 0.5390625, -0.7587890625, -1.6162109375, -0.85107421875, -0.1337890625, -0.50146484375, -0.6435546875, 1.0146484375, -2.01953125, 0.00518798828125, -2.478515625, -2.119140625, -1.216796875, 1.6669921875, 1.443359375, 0.1077880859375, -0.385498046875, 2.208984375, -0.346923828125, -1.2421875, 0.73828125, -0.802734375, 1.90234375, 0.21044921875, 0.5830078125, 0.5849609375, 1.072265625, 0.6337890625, -0.80712890625, 0.86474609375, -0.318603515625, -1.34375, -0.1954345703125, -0.2274169921875, 0.91162109375, 1.08203125, -0.60986328125, -0.66943359375, 0.9052734375, -1.2470703125, -0.329345703125, 0.8046875, 0.951171875, -1.1142578125, -0.59912109375, -0.83251953125, 0.82861328125, 0.8623046875, 0.6142578125, 0.353271484375, 0.4931640625, -0.1602783203125, 1.2548828125, 1.6611328125, -0.57568359375, -1.84375, 0.51025390625, 0.52001953125, 0.56396484375, 1.056640625, -0.1005859375, 0.459716796875, 0.26220703125, 0.59423828125, -0.94287109375, -0.1834716796875, -0.461181640625, -0.023529052734375, 0.521484375, -2.484375, -0.95703125, 1.03515625, 1.013671875, -0.17578125, 1.5966796875, 0.2052001953125, 0.6201171875, 0.24755859375, -1.220703125, -1.4697265625, 1.84375, 0.2210693359375, -1.876953125, -1.7177734375, -1.0087890625, -0.17919921875, 0.2418212890625, 1.021484375, 0.34814453125, 1.0322265625, 0.216552734375, 0.6796875, 0.254150390625, -0.849609375, -1.197265625, -0.7109375, -0.060638427734375, -0.59716796875, -0.32373046875, 1.1640625, -1.552734375, 0.73876953125, -0.5439453125, -0.83203125, 1.23828125, 0.95947265625, 1.1259765625, 0.479248046875, -0.80615234375, 1.775390625, -0.04205322265625, -0.487548828125, 0.92138671875, 1.240234375, 0.12060546875, 1.8203125, -0.350830078125, -0.5927734375, -0.47900390625, 1.068359375, -0.1982421875, -0.72314453125, -0.8125, 1.3798828125, -1.080078125, -1.0947265625, -0.0869140625, -0.45654296875, 1.73046875, -0.6142578125, 0.4677734375, 1.0390625, -1.65625, 2.349609375, -1.3623046875, 0.66845703125, -1.9775390625, -1.06640625, 0.1439208984375, 0.63427734375, -0.96875, -1.3271484375, -0.861328125, 0.71630859375, 0.7060546875, -0.13330078125, 0.61181640625, 0.12890625, 0.0157928466796875, 1.705078125, -0.361328125, -0.52880859375, -0.29736328125, 0.9521484375, -1.728515625, -2.181640625, -0.3505859375, -0.68505859375, -1.1552734375, 0.170654296875, -1.04296875, -0.39794921875, -0.78564453125, -1.001953125, -0.01355743408203125, 0.4775390625, -0.242919921875, -0.67041015625, 0.20947265625, -1.1240234375, 3.130859375, 2.50390625, -1.6015625, -1.2919921875, 1.345703125, 0.96826171875, -0.1556396484375, 1.6943359375, -1.09375, 0.12158203125, 1.0810546875, -1.6611328125, -0.314697265625, 1.0244140625, 0.6083984375, 0.6923828125, 0.252685546875, 0.5927734375, 0.352783203125, -0.43115234375, 0.71875, 0.62451171875, 0.5654296875, 0.320556640625, 0.44140625, 0.389404296875, 1.0400390625, 0.26806640625, -0.83837890625, -1.8310546875, -0.65576171875, 0.447021484375, -0.08270263671875, -1.072265625, -1.783203125, -0.11474609375, -1.3505859375, 1.615234375, -1.49609375, -0.55126953125, -0.845703125, -1.169921875, -0.327392578125, -2.05859375, 1.390625, 2.19140625, 1.041015625, 0.315185546875, -0.77685546875, 0.40966796875, 0.46533203125, -0.7412109375, -0.2578125, -0.89501953125, 1.3544921875, 0.48828125, -0.40185546875, -0.1260986328125, -0.72216796875, -0.07672119140625, 0.5908203125, -0.96435546875, 0.469482421875, 0.11175537109375, -0.54443359375, -0.6044921875, 2.046875, 0.81298828125, 1.2412109375, -1.2353515625, -0.472412109375, 0.333251953125, -0.1295166015625, -0.1068115234375, -0.85498046875, -0.241455078125, 0.85400390625, -0.09820556640625, 1.3427734375, -0.339599609375, 0.481201171875, -0.0214691162109375, -0.89697265625, -0.401123046875, -0.059234619140625, 0.358154296875, 0.41162109375, -0.292236328125, 0.8505859375, -0.09368896484375, 1.3017578125, 0.86962890625, 0.4365234375, 1.3671875, 1.666015625, -0.25732421875, 1.16015625, -0.5888671875, 1.0849609375, -0.174560546875, -0.5986328125, -0.3515625, -1.6591796875, 0.60595703125, -1.353515625, -0.53955078125, -0.8466796875, 0.34033203125, 0.0736083984375, 0.35498046875, 0.67138671875, -0.2244873046875, 0.6904296875, -0.5615234375, -1.234375, 0.5048828125, -1.0, -0.298583984375, 1.064453125, -0.69091796875, 0.5498046875, 1.3701171875, -1.2158203125, 0.5322265625, -0.59765625, 0.7607421875, 0.728515625, 0.421142578125, -0.6953125, -0.203125, 1.068359375, 0.4130859375, 0.5419921875, -2.359375, 1.697265625, -0.28857421875, -0.9619140625, -0.0164794921875, 1.10546875, 0.1571044921875, 0.46435546875, -0.435546875, 0.8046875, 0.60546875, -1.64453125, -0.30029296875, -0.5927734375, -1.6533203125, -1.5703125, 0.61865234375, -0.5791015625, 0.7421875, -0.74267578125, 2.599609375, 0.90283203125, 1.2041015625, 0.935546875, -0.407470703125, 0.08935546875, 0.93408203125, -1.43359375, -0.51611328125, -0.62841796875, 1.1201171875, 0.56640625, -1.1435546875, 0.425048828125, -1.150390625, 0.6533203125, 0.3994140625, -0.1038818359375, -0.0191497802734375, -1.2109375, 1.2802734375, -0.68115234375, -0.5234375, -0.61669921875, 1.232421875, 1.6083984375, -1.72265625, 1.7822265625, 0.0909423828125, 0.7900390625, -0.2493896484375, -0.229248046875, 0.15234375, -1.1875, -1.0634765625}; diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul copy.c b/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul copy.c index f022f21b7..a85523b20 100644 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul copy.c +++ b/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul copy.c @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Author: Pei-Yu Lin +// Author: Pei-Yu Lin, ETH Zürich // Specialized FP16 VME matmul kernel for M=N=64, K=64 or K=128. // TE=16, CE=8, and eight logical tiles. diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.c b/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.c index f022f21b7..a85523b20 100644 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.c +++ b/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.c @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Author: Pei-Yu Lin +// Author: Pei-Yu Lin, ETH Zürich // Specialized FP16 VME matmul kernel for M=N=64, K=64 or K=128. // TE=16, CE=8, and eight logical tiles. diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.h b/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.h index 8109bc713..8a7914c7b 100644 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.h +++ b/sw/vme/apps/tew16/fp16-matmul-vtse/kernel/matmul.h @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Author: Pei-Yu Lin +// Author: Pei-Yu Lin, ETH Zürich #pragma once @@ -23,7 +23,7 @@ enum { CE = 8, TE = 16, - CHUNK_SIZE = 64, + CHUNK_SIZE = 4 * TE, M_BLOCK = 4 * TE, N_BLOCK = 2 * TE, A_TILES_PER_BLOCK = M_BLOCK / TE, @@ -38,13 +38,6 @@ static inline uint32_t get_cycle(void) return c; } -// Reading fcsr stalls Snitch while any Spatz scoreboard entry is live. Use it -// to turn an asynchronous kernel return into an architectural completion point. -static inline void wait_spatz(void) -{ - uint32_t fcsr; - asm volatile("csrr %0, fcsr" : "=r"(fcsr) :: "memory"); -} // FP16 VME matmul over TE=16. One logical block is 4*TE by 2*TE: // diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/main copy.c b/sw/vme/apps/tew16/fp16-matmul-vtse/main copy.c deleted file mode 100644 index 85cf3c777..000000000 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/main copy.c +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin - -#include -#include -#include - -#include DATAHEADER -#include "kernel/gemm.c" - -enum { - TCDM_BYTES = 128 * 1024, - M_CHUNK = 64, - N_CHUNK = 64, - K_CHUNK = 128, - TE = 16, - KMAX = 2, - TILE_BYTES = TE * K_CHUNK * sizeof(__fp16), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_BYTES = M_CHUNK * N_CHUNK * sizeof(__fp16), - A_OFFSET = 0, - B_OFFSET = A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - CHUNK_SIZE = M_CHUNK * N_CHUNK, // for what - BUFFER_BYTES = C_OFFSET + C_BYTES, - BUFFER_COUNT = 2, - BUFFER_STRIDE = TCDM_BYTES / BUFFER_COUNT, - L1_BYTES = BUFFER_STRIDE + BUFFER_BYTES, -} - -_Static_assert(BUFFER_BYTES <= BUFFER_STRIDE, - "one M64xN64xK128 buffer must fit in half of TCDM"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "ping-pong GEMM buffers exceed TCDM"); - -static const __fp16 zero_tile_dram[TILE_INPUTS] - __attribute__((section(".dram"), aligned(64))) = {0}; // why align 64? = TE? - -static uint8_t *l1_base; -static float *partials; -static float *c_out; -static uint32_t allocation_failed; -static volatile uint32_t core_cycles[2]; -static volatile uint32_t verify_errors[2]; -static volatile float verify_max_abs[2]; - -// static const float *Apack, *Bpack, *Cref; -// static float *C; - -static snrt_dma_txid_t fill_panel(__fp16 *dst, const __fp16 *src, - uint32_t src_tiles, uint32_t dst_tiles, - uint32_t dst_K, uint32_t src_K) { - if (dst_tiles == src_tiles && dst_K == src_K) - return snrt_dma_start_1d(dst, src, src_tiles * TILE_BYTES); - const snrt_dma_txid_t clear_tid = - snrt_dma_start_2d(dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, src_tiles); - if (dst_tiles == 0 || dst_K == 0) return clear_tid; - - const size_t copy_bytes = dst_K * TE * sizeof(__fp16); - const size_t stride = src_K * TE * sizeof(__fp16); - return snrt_dma_start_2d(dst, src, copy_bytes, TILE_BYTES, stride, dst_tiles); -} - -static inline uint32_t min_u32(uint32_t a, uint32_t b) { - return a < b ? a : b; -} - -static inline uint32_t chunk_extent(uint32_t total, uint32_t block, - uint32_t chunk) { - const uint32_t offset = block * chunk; - return offset < total ? min_u32(chunk, total - offset) : 0; -} - -static snrt_dma_txid_t fill_buffer(unit32_t buffer, unit32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * BUFFER_STRIDE; - const uint32_t k0 = kb * K_CHUNK; - const uint32_t src_K = chunk_extent(gemm_l.K, kb, K_CHUNK); - const uint32_t m_tiles = (gemm_l.M + TE - 1) / TE; - const uint32_t n_tiles = (gemm_l.N + TE - 1) / TE; - const uint32_t first_mt = mb * A_TILES; - const uint32_t first_nt = nb * B_TILES; - const uint32_t src_a_tiles = first_mt < m_tiles ? min_u32(A_TILES, m_tiles - first_mt) : 0; - const uint32_t src_b_tiles = first_nt < n_tiles ? min_u32(B_TILES, n_tiles - first_nt) : 0; - const __fp16 *src_a = src_a_tiles==0 ? zero_tile_dram - : gemm_Apack_dram + (first_mt * KMAX + k0) * TE; // KMAX? or K - const __fp16 *src_b = src_b_tiles==0 ? zero_tile_dram - : gemm_Bpack_dram + (first_nt * KMAX + k0) * TE; - const __fp16 *dst_a = (__fp16 *)(base + A_OFFSET); - const __fp16 *dst_b = (__fp16 *)(base + B_OFFSET); - - // fill A tile - fill_panel(dst_a, src_a, A_TILES, src_a_tiles, src_K, KMAX); // KMAX or K? - // fill B tile - return fill_panel(dst_b, src_b, B_TILES, src_b_tiles, src_K, KMAX); -} - -int main(void) -{ - const uint32_t cid = snrt_cluster_core_idx(); - const uint32_t ncores = snrt_cluster_core_num(); -#ifdef SPATZ_GEMM_SINGLE_CORE - const uint32_t active_cores = 1; -#else - const uint32_t active_cores = ncores; -#endif - - const uint32_t row_pairs = M_CHUNK / (2 * TE); - const uint32_t mb_count = (gemm_l.M + M_CHUNK - 1) / M_CHUNK; - const uint32_t nb_count = (gemm_l.N + N_CHUNK - 1) / N_CHUNK; - const uint32_t kb_count = (gemm_l.K + K_CHUNK - 1) / K_CHUNK; - const uint32_t jobs = mb_count * nb_count; - const uint32_t iterations = jobs * kb_count; - - if (cid == 0) { - // core 0 has DMA - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (__fp16 *)snrt_l3alloc(iterations * CHUNK_SIZE * sizeof(__fp16)); // variable? TEW - c_out = (float *)snrt_l3alloc(TE * TE * sizeof(__fp16)); - allocation_failed = l1_base == 0 || partials == 0 || c_out == 0; - } - // snrt_cluster_hw_barrier(); - snrt_dma_txid_t input_tid[BUFFER_COUNT] = {0, 0}; - - uint32_t current_buffer = 0; - uint32_t kernel_cycles = 0; - - if (cid == 0) { - input_tid[0] = fill_buffer(0, 0, 0, 0); - snrt_dma_wait(input_tid[0]); - } - // snrt_cluster_hw_barrier(); - - // Iter 0 computes from buffer 0 while this descriptor chain fills buffer 1. - if (cid == 0 && iterations > 1) { - const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, next_job % nb_count, 1 % kb_count); - } - - for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = job % nb_count; - const uint32_t valid_m = chunk_extent(gemm_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(gemm_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - const uint32_t next_iter = iter + 1; - const uint32_t has_next = next_iter < iterations; - const uint32_t refill_iter = iter + 2; - const uint32_t has_refill = refill_iter < iterations; - const uint32_t next_buffer = current_buffer ^ 1; - - if (cid < active_cores) { - uint8_t *base = l1_base + current_buffer * BUFFER_STRIDE; - __fp16 *a = (__fp16 *)(base + A_OFFSET); - __fp16 *b = (__fp16 *)(base + B_OFFSET); - __fp16 *c = (__fp16 *)(base + C_OFFSET); - const uint32_t t0 = gemm_cycle(); - uint32_t did_compute = 0; - // since only use core 0 - // TODO: remove m pair, for-loop, remove finish_spatz(), replace core_m, core_row concept - for (uint32_t pair = cid; pair < row_pairs; - pair += active_cores) { - const uint32_t core_row = 2 * TILE_DIM; - const uint32_t core_m = core_row < valid_m - ? min_u32(2 * TILE_DIM, - valid_m - core_row) - : 0; - if (core_m != 0) { - gemm_block_fp16( - (void *)((uint8_t *)a + - core_row * K_CHUNK * sizeof(__fp16)), - (void *)b, - (void *)((uint8_t *)c + - core_row * N_CHUNK * sizeof(float)), - (int)valid_k, (int)valid_n, (int)core_m, 0); - did_compute = 1; - } - } - - if (did_compute) { - finish_spatz(); - kernel_cycles += gemm_cycle() - t0; - } - } - if (cid == 0) { - // Do not switch to the alternate buffer until all of its A/B - // descriptors have completed. - if (has_next) - snrt_dma_wait(input_tid[next_buffer]); - - float *src = (float *)(l1_base + current_buffer * BUFFER_STRIDE + C_OFFSET); - snrt_dma_start_1d(partials + iter * PARTIAL_FLOATS, src, C_BYTES); - - // Reuse the just-computed buffer for iteration i+2. The DMA - // executes descriptors in issue order, so C(i) is read before the - // following A/B refill overwrites this 64 KiB half. This complete - // chain overlaps compute(i+1) in the other half. - if (has_refill) { - const uint32_t refill_job = refill_iter / kb_count; - input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, - refill_job % nb_count, refill_iter % kb_count); - } - } - // gemm_barrier(active_cores); - if (has_next) - current_buffer = next_buffer; - } - if (cid < post_cores) { - verify_errors[cid] = local_errors; - verify_max_abs[cid] = local_max_abs; - } - // snrt_cluster_hw_barrier(); - - - if (cid < active_cores) - core_cycles[cid] = kernel_cycles; - - if (cid == 0) - snrt_dma_wait_all(); - // snrt_cluster_hw_barrier(); - - if (cid < post_cores) { - const uint32_t output_elements = gemm_l.M * gemm_l.N; - for (uint32_t idx = cid; idx < output_elements; idx += post_cores) { - const uint32_t row = idx / gemm_l.N; - const uint32_t col = idx % gemm_l.N; - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - float sum = 0.0f; - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float *partial = partials + - (job * kb_count + kb) * PARTIAL_FLOATS; - sum += partial_at(partial, local_row, local_col); - } - c_out[row * gemm_l.Np + col] = sum; - } - } - // snrt_cluster_hw_barrier(); - - uint32_t cycles = core_cycles[0]; - for (uint32_t core = 1; core < active_cores; ++core) { - if (core_cycles[core] > cycles) - cycles = core_cycles[core]; - } - - enum { CE = 8 }; - const uint64_t useful_fmas = - (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE * active_cores; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - uint32_t local_errors = 0; - uint32_t printed_partial_debug = 0; - float local_max_abs = 0.0f; - // FP16 inputs are accumulated in FP32, but panel reduction changes the - // FP32 addition order relative to the host reference. - const float tolerance = 5.0e-2f; - - if (cid != 0) - return 0; - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t core = 0; core < post_cores; ++core) { - errors += verify_errors[core]; - if (verify_max_abs[core] > max_abs) - max_abs = verify_max_abs[core]; - } - - printf("\n=== General FP16 GEMM %ux%ux%u " - "(M64xN64xK128 panels, 2 buffers, %u core%s) ===\n", - gemm_l.M, gemm_l.N, gemm_l.K, active_cores, - active_cores == 1 ? "" : "s"); - printf("Cycles: %u\n", cycles); - printf("Utilization: %u.%02u%%\n", utilization_bp / 100, - utilization_bp % 100); - printf("MaxAbsErr: 0x%08x\n", *(uint32_t *)&max_abs); - printf("Errors: %u\n", errors); - printf("Status: %s\n", errors ? "FAIL" : "PASS"); - return errors ? 1 : 0; -} diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/main.c b/sw/vme/apps/tew16/fp16-matmul-vtse/main.c index 7d7ec2a0f..5c60f2a9d 100644 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/main.c +++ b/sw/vme/apps/tew16/fp16-matmul-vtse/main.c @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// Author: Pei-Yu Lin +// Author: Pei-Yu Lin, ETH Zürich // main.c - fp16 x fp16 = fp16 VME GEMM functional check @@ -26,7 +26,6 @@ #include "kernel/matmul.c" enum { - TCDM_BYTES = 128 * 1024, M_CHUNK = CHUNK_SIZE, N_CHUNK = CHUNK_SIZE, K_CHUNK = CHUNK_SIZE * 2, @@ -43,14 +42,14 @@ enum { C_OFFSET = B_OFFSET + B_BYTES, BUFFER_BYTES = C_OFFSET + C_BYTES, L1_BUFFER_COUNT = 2, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, L1_BYTES = L1_BUFFER_STRIDE + BUFFER_BYTES, }; _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, - "one M64xN64xK128 TEW16 buffer must fit in 64 KiB"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "two matmul buffers exceed the 128 KiB TCDM"); + "one M64xN64xK128 TEW16 buffer must fit in 128 KiB"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "two matmul buffers exceed the configured TCDM capacity"); static const __fp16 zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -201,8 +200,6 @@ int main(void) { (const __fp16 *)(base + B_OFFSET), (__fp16 *)(base + C_OFFSET), valid_m, valid_n, valid_k); end_cycle = cycle_count(); - // Keep completion synchronization outside the measured kernel window. - wait_spatz(); snrt_dma_start_1d(partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); diff --git a/sw/vme/apps/tew16/fp16-matmul-vtse/script/gemm_64_64_64.json b/sw/vme/apps/tew16/fp16-matmul-vtse/script/gemm_64_64_64.json index 538428508..fdf0f2a30 100644 --- a/sw/vme/apps/tew16/fp16-matmul-vtse/script/gemm_64_64_64.json +++ b/sw/vme/apps/tew16/fp16-matmul-vtse/script/gemm_64_64_64.json @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0, see LICENSE for details. // SPDX-License-Identifier: Apache-2.0 // -// Author: Pei-Yu Lin +// Author: Pei-Yu Lin, ETH Zürich // // Configuration for a 64×64×64 VME GEMM benchmark. { diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/CMakeLists.txt b/sw/vme/apps/tew32/fp16-gemm-alpha/CMakeLists.txt new file mode 100644 index 000000000..1aae9d973 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +set(_source "${CMAKE_CURRENT_SOURCE_DIR}") + +foreach(_k IN ITEMS 128 256 512) + math(EXPR _a_bytes "64 * ${_k} * 2") + math(EXPR _b_bytes "${_k} * 64 * 2") + math(EXPR _c_bytes "64 * 64 * 4") + math(EXPR _required_tcdm_bytes + "${_a_bytes} + ${_b_bytes} + ${_c_bytes}") + if(_required_tcdm_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping fp16-gemm-alpha M64 N64 K${_k}: requires " + "${_required_tcdm_bytes} TCDM bytes, but only " + "${SNRT_TCDM_SIZE} are configured") + continue() + endif() + + set(_target "fp16-gemm-alpha_M64_N64_K${_k}") + set(_config "${_source}/script/gemm_64_64_${_k}.json") + set(_header "${_source}/data/data_64_64_${_k}.h") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${_source}/script/gen_data.py" + -c "${_config}" + DEPENDS "${_source}/script/gen_data.py" + "${_config}" + "${_source}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${_source}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${_source}" "${_source}/kernel" "${_source}/data") + target_compile_definitions(${_test_target} PRIVATE + K_CHUNK=${_k} + DATAHEADER="data/data_64_64_${_k}.h") + endif() +endforeach() diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-alpha/data/layer.h new file mode 100644 index 000000000..d80aae389 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME GEMM layer configuration. + * + * Describes C = alpha * (A * B) + bias with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} gemm_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/kernel/gemm.c b/sw/vme/apps/tew32/fp16-gemm-alpha/kernel/gemm.c new file mode 100644 index 000000000..5736760b6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/kernel/gemm.c @@ -0,0 +1,200 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// FP16-to-FP32 GEMM kernel for TE=16, CE=8, and four accumulator tiles. + +#include "gemm.h" + +__attribute__((noinline, aligned(64))) void gemm_fp16_fp32_alpha_vle( + const __fp16 *Atpack, const __fp16 *Bpack, float *C, + const float *alpha) +{ + register float alpha_ft0 asm("ft0"); + asm volatile("flw %0, 0(%1)" : "=f"(alpha_ft0) : "r"(alpha) : "memory"); + + const uintptr_t c_stride = 128 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t vtype = 0xC8; // e16, m1, tk=2 widening SDOTP + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + const uintptr_t bias_vl = 8 * TE; // m8: two contiguous 64-element FP32 rows + const uintptr_t operand_vector_bytes = TE * sizeof(uint32_t); + const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + + // Process each 16-row output block independently; no state crosses blocks. + for (uint32_t nb = 0; nb < 4; nb += 1) { + const uint32_t mb0 = nb * TE; + { + // Initialize and compute one complete 16-row output block. + uintptr_t a0p = (uintptr_t)(Atpack + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(Bpack + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(Bpack + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t bias_ptr = ptr; + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtzero mt0" : : : "memory"); + asm volatile("vtzero mt4" : : : "memory"); + asm volatile("vtzero mt8" : : : "memory"); + asm volatile("vtzero mt12" : : : "memory"); + + // Use the already-advanced A pointer as the loop induction variable. + const uintptr_t a_middle_end = a0p + (K_CHUNK / 8 - 1) * operand_group_bytes; + while (a0p != a_middle_end) { + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vle32.v v24, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v24", "memory"); + asm volatile("vle32.v v25, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v25", "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vle32.v v26, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v26", "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("vle32.v v27, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v27", "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + // Keep the pointer comparison; otherwise Clang adds a separate countdown. + asm volatile("" : "+r"(a0p)); + } + + // Final K group: move each completed tile row to the vector registers. + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b1p) : "v24", "v25", "v26", "v27", "memory"); b1p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vle32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfmacc.vf v0, %0, v8" : : "f"(alpha_ft0) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vfmacc.vf v24, %0, v16" : : "f"(alpha_ft0) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("vse32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vse32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + + // Rows 4-15 + for (uint32_t row = 4; row < TE; row += 4) { + asm volatile("vle32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfmacc.vf v0, %0, v8" : : "f"(alpha_ft0) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vle32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfmacc.vf v24, %0, v16" : : "f"(alpha_ft0) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("vse32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vse32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + } + } + } +} diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/kernel/gemm.h b/sw/vme/apps/tew32/fp16-gemm-alpha/kernel/gemm.h new file mode 100644 index 000000000..c66faa992 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/kernel/gemm.h @@ -0,0 +1,17 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +void gemm_fp16_fp32_alpha_vle(const __fp16 *Atpack, const __fp16 *Bpack, + float *C, const float *alpha); diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/main.c b/sw/vme/apps/tew32/fp16-gemm-alpha/main.c new file mode 100644 index 000000000..9dc98c451 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/main.c @@ -0,0 +1,100 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include "gemm.h" +#include DATAHEADER +#include "kernel/gemm.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle) : : "memory"); + return cycle; +} + +static inline uint32_t float_bits(float value) { + union { float f; uint32_t u; } bits = {.f = value}; + return bits.u; +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + if (gemm_l.M != 64 || gemm_l.N != 64 || + (gemm_l.K != 128 && gemm_l.K != 256 && gemm_l.K != 512)) { + printf("Unsupported GEMM shape: expected M64_N64_K128, K256, or K512\n"); + return 2; + } + + enum { + A_BYTES = sizeof(Atpack_dram), + B_BYTES = sizeof(Bpack_dram), + C_BYTES = sizeof(Caddend_dram), + SCALER_BYTES = sizeof(scaler_dram), + BUFFER_BYTES = A_BYTES + B_BYTES + C_BYTES + SCALER_BYTES + }; + _Static_assert(BUFFER_BYTES <= SNRT_TCDM_SIZE, "GEMM buffers exceed TCDM"); + + uint8_t *const buffer = (uint8_t *)snrt_l1alloc(BUFFER_BYTES); + if (buffer == 0) + return 2; + + __fp16 *const Atpack = (__fp16 *)buffer; + __fp16 *const Bpack = (__fp16 *)(buffer + A_BYTES); + float *const C = (float *)(buffer + A_BYTES + B_BYTES); + float *const scaler = (float *)(buffer + A_BYTES + B_BYTES + C_BYTES); + + snrt_dma_start_1d(Atpack, Atpack_dram, A_BYTES); + snrt_dma_start_1d(Bpack, Bpack_dram, B_BYTES); + snrt_dma_start_1d(C, Caddend_dram, C_BYTES); + snrt_dma_start_1d(scaler, &scaler_dram, SCALER_BYTES); + snrt_dma_wait_all(); + + const uint32_t start = get_cycle(); + gemm_fp16_fp32_alpha_vle(Atpack, Bpack, C, scaler); + const uint32_t cycles = get_cycle() - start; + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < gemm_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < gemm_l.N; ++col) { + const uint32_t index = row * gemm_l.N + col; + const float expected = Cref_dram[index]; + const float delta = C[index] > expected ? C[index] - expected + : expected - C[index]; + const float tolerance = 0.001f + 0.001f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols %u", row, col); + else + printf(",%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint64_t useful_macs = + (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; + const uint64_t peak_macs = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization = + peak_macs ? useful_macs * 10000 / peak_macs : 0; + printf("\n=== TEW32 FP16 scaled GEMM M%u_N%u_K%u " + "(TE=16, 1 core) ===\n", + gemm_l.M, gemm_l.N, gemm_l.K); + printf("Scaler: 0x%08x\n", float_bits(*scaler)); + printf("Cycles: %u\nUtilization: %u.%02u%%\n", + cycles, utilization / 100, utilization % 100); + printf("MaxAbsErr: 0x%08x\nErrors: %u\nStatus: %s\n", + float_bits(max_abs), errors, errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_128.json new file mode 100644 index 000000000..1f38616b3 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_128.json @@ -0,0 +1,10 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + scaler: 0.5, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_256.json new file mode 100644 index 000000000..c8ae0f991 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_256.json @@ -0,0 +1,10 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + scaler: 0.5, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_512.json new file mode 100644 index 000000000..2a4d75bb7 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gemm_64_64_512.json @@ -0,0 +1,10 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + scaler: 0.5, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-alpha/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gen_data.py new file mode 100644 index 000000000..4ed4bc485 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-alpha/script/gen_data.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin, ETH Zürich +# +# +# Usage: +# python3 script/gen_data.py -c script/gemm_64_64_64.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Required fields: TM, TN; optional fields: scaler (default 0.5), seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a, b, te): + """Pack adjacent K positions into the two FP16 halves of one word.""" + m, k = a.shape + n = b.shape[1] + abits = a.view(np.uint16).astype(np.uint32) + bbits = b.view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate packed FP16-to-FP32 scaled GEMM data" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + TM = int(param["TM"]) + TN = int(param["TN"]) + scaler = np.float32(param.get("scaler", 0.5)) + seed = int(param.get("seed", 42)) + + if M != 64 or N != 64 or K not in (128, 256, 512) or (TM, TN) != (16, 16): + parser.error("this test targets M64_N64_K128, K256, or K512 with TE=16") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16).numpy() + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16).numpy() + bias = torch.randn((M, N), dtype=torch.float32).numpy() + + # Match SDOTP: each adjacent K pair is reduced and accumulated into FP32. + product = np.zeros((M, N), dtype=np.float32) + for k in range(0, K, 2): + pair = (A[:, k, None].astype(np.float64) * B[None, k, :].astype(np.float64) + + A[:, k + 1, None].astype(np.float64) * B[None, k + 1, :].astype(np.float64)) + product = (product.astype(np.float64) + pair).astype(np.float32) + + # Model the FP32 vfmacc.vf: one final rounding of bias + scaler * product. + C = (bias.astype(np.float64) + + np.float64(scaler) * product.astype(np.float64)).astype(np.float32) + A_pack, B_pack = pack_pairs(A, B, TM) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin, ETH Zürich", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + "", + "const gemm_layer gemm_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 scalar alpha in C = alpha * (A*B) + bias.", + "static float scaler_dram __attribute__((section(\".data\"), aligned(4))) =", + f" {repr(float(scaler))}f;", + "", + "// FP32 golden reference: C = scaler * (A*B) + bias.", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// FP32 bias, with one independent value per output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(bias) + ";", + "", + "// Atpack[m_tile][K/2][TE]: high half even K, low half odd K.", + f'static uint32_t Atpack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// Bpack[n_tile][K/2][TE], matching Atpack halfword order.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/CMakeLists.txt b/sw/vme/apps/tew32/fp16-gemm-beta/CMakeLists.txt new file mode 100644 index 000000000..7da848fef --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/CMakeLists.txt @@ -0,0 +1,47 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +set(_source "${CMAKE_CURRENT_SOURCE_DIR}") + +foreach(_k IN ITEMS 128 256 512) + math(EXPR _a_bytes "64 * ${_k} * 2") + math(EXPR _b_bytes "${_k} * 64 * 2") + math(EXPR _c_bytes "64 * 64 * 4") + math(EXPR _required_tcdm_bytes + "${_a_bytes} + ${_b_bytes} + ${_c_bytes}") + if(_required_tcdm_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping fp16-gemm-beta M64 N64 K${_k}: requires " + "${_required_tcdm_bytes} TCDM bytes, but only " + "${SNRT_TCDM_SIZE} are configured") + continue() + endif() + + set(_target "fp16-gemm-beta_M64_N64_K${_k}") + set(_config "${_source}/script/gemm_64_64_${_k}.json") + set(_header "${_source}/data/data_64_64_${_k}.h") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${_source}/script/gen_data.py" + -c "${_config}" + DEPENDS "${_source}/script/gen_data.py" + "${_config}" + "${_source}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${_source}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${_source}" "${_source}/kernel" "${_source}/data") + target_compile_definitions(${_test_target} PRIVATE + K_CHUNK=${_k} + DATAHEADER="data/data_64_64_${_k}.h") + endif() +endforeach() + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-beta/data/layer.h new file mode 100644 index 000000000..0c67fdf4c --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/data/layer.h @@ -0,0 +1,31 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME GEMM layer configuration. + * + * Describes C = alpha * (A * B) + beta * bias with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} gemm_layer; + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/kernel/gemm.c b/sw/vme/apps/tew32/fp16-gemm-beta/kernel/gemm.c new file mode 100644 index 000000000..376063763 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/kernel/gemm.c @@ -0,0 +1,213 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// FP16-to-FP32 alpha-beta GEMM kernel for TE=16, CE=8, and four accumulator tiles. + +#include "gemm.h" + +__attribute__((noinline, aligned(64))) void gemm_fp16_fp32_beta_vle( + const __fp16 *Atpack, const __fp16 *Bpack, float *C, + const float *alpha, const float *beta) +{ + register float alpha_ft0 asm("ft0"); + register float beta_ft1 asm("ft1"); + asm volatile("flw %0, 0(%2)\nflw %1, 0(%3)" + : "=f"(alpha_ft0), "=f"(beta_ft1) + : "r"(alpha), "r"(beta) + : "memory"); + + const uintptr_t c_stride = 128 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t vtype = 0xC8; // e16, m1, tk=2 widening SDOTP + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + const uintptr_t bias_vl = 8 * TE; // m8: two contiguous 64-element FP32 rows + const uintptr_t operand_vector_bytes = TE * sizeof(uint32_t); + const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + + // Process each 16-row output block independently; no state crosses blocks. + for (uint32_t nb = 0; nb < 4; nb += 1) { + const uint32_t mb0 = nb * TE; + { + // Initialize and compute one complete 16-row output block. + uintptr_t a0p = (uintptr_t)(Atpack + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(Bpack + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(Bpack + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t bias_ptr = ptr; + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtzero mt0" : : : "memory"); + asm volatile("vtzero mt4" : : : "memory"); + asm volatile("vtzero mt8" : : : "memory"); + asm volatile("vtzero mt12" : : : "memory"); + + // Use the already-advanced A pointer as the loop induction variable. + const uintptr_t a_middle_end = a0p + (K_CHUNK / 8 - 1) * operand_group_bytes; + while (a0p != a_middle_end) { + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vle32.v v24, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v24", "memory"); + asm volatile("vle32.v v25, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v25", "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vle32.v v26, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v26", "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("vle32.v v27, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v27", "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + // Keep the pointer comparison; otherwise Clang adds a separate countdown. + asm volatile("" : "+r"(a0p)); + } + + // Final K group: move each completed tile row to the vector registers. + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b1p) : "v24", "v25", "v26", "v27", "memory"); b1p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + // mt0 has been moved + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + // mt4 has been moved + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + // mt8 has been moved + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + // mt12 has been moved + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfmul.vf v0, v0, %0" : : "f"(beta_ft1) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vfmacc.vf v0, %0, v8" : : "f"(alpha_ft0) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vle32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("vse32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vfmul.vf v24, v24, %0" : : "f"(beta_ft1) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("vfmacc.vf v24, %0, v16" : : "f"(alpha_ft0) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + + // Rows 4-15 + for (uint32_t row = 4; row < TE; row += 4) { + asm volatile("vle32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vse32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vfmul.vf v0, v0, %0" : : "f"(beta_ft1) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vfmacc.vf v0, %0, v8" : : "f"(alpha_ft0) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vle32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vse32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vfmul.vf v24, v24, %0" : : "f"(beta_ft1) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("vfmacc.vf v24, %0, v16" : : "f"(alpha_ft0) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + } + asm volatile("vse32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + } + } +} + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/kernel/gemm.h b/sw/vme/apps/tew32/fp16-gemm-beta/kernel/gemm.h new file mode 100644 index 000000000..59f733ca1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/kernel/gemm.h @@ -0,0 +1,19 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +void gemm_fp16_fp32_beta_vle(const __fp16 *Atpack, const __fp16 *Bpack, + float *C, const float *alpha, + const float *beta); + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/main.c b/sw/vme/apps/tew32/fp16-gemm-beta/main.c new file mode 100644 index 000000000..73122ce8d --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/main.c @@ -0,0 +1,107 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include "gemm.h" +#include DATAHEADER +#include "kernel/gemm.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle) : : "memory"); + return cycle; +} + +static inline uint32_t float_bits(float value) { + union { float f; uint32_t u; } bits = {.f = value}; + return bits.u; +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + if (gemm_l.M != 64 || gemm_l.N != 64 || + (gemm_l.K != 128 && gemm_l.K != 256 && gemm_l.K != 512)) { + printf("Unsupported GEMM shape: expected M64_N64_K128, K256, or K512\n"); + return 2; + } + + enum { + A_BYTES = sizeof(Atpack_dram), + B_BYTES = sizeof(Bpack_dram), + C_BYTES = sizeof(Caddend_dram), + ALPHA_BYTES = sizeof(alpha_dram), + BETA_BYTES = sizeof(beta_dram), + BUFFER_BYTES = A_BYTES + B_BYTES + C_BYTES + + ALPHA_BYTES + BETA_BYTES + }; + _Static_assert(BUFFER_BYTES <= SNRT_TCDM_SIZE, "GEMM buffers exceed TCDM"); + + uint8_t *const buffer = (uint8_t *)snrt_l1alloc(BUFFER_BYTES); + if (buffer == 0) + return 2; + + __fp16 *const Atpack = (__fp16 *)buffer; + __fp16 *const Bpack = (__fp16 *)(buffer + A_BYTES); + float *const C = (float *)(buffer + A_BYTES + B_BYTES); + float *const alpha = (float *)(buffer + A_BYTES + B_BYTES + C_BYTES); + float *const beta = (float *)(buffer + A_BYTES + B_BYTES + C_BYTES + + ALPHA_BYTES); + + snrt_dma_start_1d(Atpack, Atpack_dram, A_BYTES); + snrt_dma_start_1d(Bpack, Bpack_dram, B_BYTES); + snrt_dma_start_1d(C, Caddend_dram, C_BYTES); + snrt_dma_start_1d(alpha, &alpha_dram, ALPHA_BYTES); + snrt_dma_start_1d(beta, &beta_dram, BETA_BYTES); + snrt_dma_wait_all(); + + const uint32_t start = get_cycle(); + gemm_fp16_fp32_beta_vle(Atpack, Bpack, C, alpha, beta); + const uint32_t cycles = get_cycle() - start; + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < gemm_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < gemm_l.N; ++col) { + const uint32_t index = row * gemm_l.N + col; + const float expected = Cref_dram[index]; + const float delta = C[index] > expected ? C[index] - expected + : expected - C[index]; + const float tolerance = 0.001f + 0.001f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols %u", row, col); + else + printf(",%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint64_t useful_macs = + (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; + const uint64_t peak_macs = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization = + peak_macs ? useful_macs * 10000 / peak_macs : 0; + printf("\n=== TEW32 FP16 alpha-beta GEMM M%u_N%u_K%u " + "(TE=16, 1 core) ===\n", + gemm_l.M, gemm_l.N, gemm_l.K); + printf("Alpha: 0x%08x\nBeta: 0x%08x\n", + float_bits(*alpha), float_bits(*beta)); + printf("Cycles: %u\nUtilization: %u.%02u%%\n", + cycles, utilization / 100, utilization % 100); + printf("MaxAbsErr: 0x%08x\nErrors: %u\nStatus: %s\n", + float_bits(max_abs), errors, errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_128.json new file mode 100644 index 000000000..4e4b6741a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_128.json @@ -0,0 +1,12 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + alpha: 0.5, + beta: 0.25, + seed: 42 +} + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_256.json new file mode 100644 index 000000000..ee2631bf9 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_256.json @@ -0,0 +1,12 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + alpha: 0.5, + beta: 0.25, + seed: 42 +} + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_512.json new file mode 100644 index 000000000..f44ab38eb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/script/gemm_64_64_512.json @@ -0,0 +1,12 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + alpha: 0.5, + beta: 0.25, + seed: 42 +} + diff --git a/sw/vme/apps/tew32/fp16-gemm-beta/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-beta/script/gen_data.py new file mode 100644 index 000000000..3fe2f8555 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-beta/script/gen_data.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin, ETH Zürich +# +# +# Usage: +# python3 script/gen_data.py -c script/gemm_64_64_64.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Required fields: TM, TN; optional fields: alpha (default 0.5), beta (default 0.25), seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a, b, te): + """Pack adjacent K positions into the two FP16 halves of one word.""" + m, k = a.shape + n = b.shape[1] + abits = a.view(np.uint16).astype(np.uint32) + bbits = b.view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate packed FP16-to-FP32 alpha-beta GEMM data" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + TM = int(param["TM"]) + TN = int(param["TN"]) + alpha = np.float32(param.get("alpha", 0.5)) + beta = np.float32(param.get("beta", 0.25)) + seed = int(param.get("seed", 42)) + + if M != 64 or N != 64 or K not in (128, 256, 512) or (TM, TN) != (16, 16): + parser.error("this test targets M64_N64_K128, K256, or K512 with TE=16") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16).numpy() + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16).numpy() + bias = torch.randn((M, N), dtype=torch.float32).numpy() + + # Match SDOTP: each adjacent K pair is reduced and accumulated into FP32. + product = np.zeros((M, N), dtype=np.float32) + for k in range(0, K, 2): + pair = (A[:, k, None].astype(np.float64) * B[None, k, :].astype(np.float64) + + A[:, k + 1, None].astype(np.float64) * B[None, k + 1, :].astype(np.float64)) + product = (product.astype(np.float64) + pair).astype(np.float32) + + # Match the direct kernel: fma(alpha, product, round_fp32(beta * bias)). + scaled_bias = (np.float64(beta) * bias.astype(np.float64)).astype(np.float32) + C = (scaled_bias.astype(np.float64) + + np.float64(alpha) * product.astype(np.float64)).astype(np.float32) + A_pack, B_pack = pack_pairs(A, B, TM) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin, ETH Zürich", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + "", + "const gemm_layer gemm_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 scalars in C = alpha * (A*B) + beta * bias.", + "static float alpha_dram __attribute__((section(\".data\"), aligned(4))) =", + f" {repr(float(alpha))}f;", + "static float beta_dram __attribute__((section(\".data\"), aligned(4))) =", + f" {repr(float(beta))}f;", + "", + "// FP32 golden reference: C = alpha * (A*B) + beta * bias.", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// FP32 bias, with one independent value per output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(bias) + ";", + "", + "// Atpack[m_tile][K/2][TE]: high half even K, low half odd K.", + f'static uint32_t Atpack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// Bpack[n_tile][K/2][TE], matching Atpack halfword order.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() + diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/.gitignore b/sw/vme/apps/tew32/fp16-gemm-softmax/.gitignore new file mode 100644 index 000000000..718d01535 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/.gitignore @@ -0,0 +1,2 @@ +data/data_*.h +__pycache__/ diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/CMakeLists.txt b/sw/vme/apps/tew32/fp16-gemm-softmax/CMakeLists.txt new file mode 100644 index 000000000..2d53cd076 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +if(BUILD_TESTS AND "te16" IN_LIST VME_TILE_VARIANT) + foreach(_k IN ITEMS 128 256 512) + set(_target "fp16-gemm-softmax-te16-rolled_M64_N64_K${_k}") + set(_header "${CMAKE_CURRENT_SOURCE_DIR}/data/data_64_64_${_k}.h") + set(_config "${CMAKE_CURRENT_BINARY_DIR}/matmul_64_64_${_k}.json") + file(WRITE "${_config}" "{ M: 64, N: 64, K: ${_k}, TM: 16, TN: 16, seed: 42 }\n") + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py" -c "${_config}" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py" "${_config}" + "${CMAKE_CURRENT_SOURCE_DIR}/data/layer.h" + VERBATIM) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${CMAKE_CURRENT_SOURCE_DIR}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/kernel") + target_compile_options(${_test_target} PRIVATE -fno-vectorize -fno-slp-vectorize) + target_compile_definitions(${_test_target} PRIVATE + MATMUL_BUFFER_COUNT=1 DATAHEADER="data/data_64_64_${_k}.h") + endforeach() +endif() diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/README.md b/sw/vme/apps/tew32/fp16-gemm-softmax/README.md new file mode 100644 index 000000000..0cc493ba6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/README.md @@ -0,0 +1,65 @@ +# FP16 GEMM + column softmax, 1x4 + +Computes `softmax(A @ B + bias, dim=1).T`, stored as FP32 +`[vocab=64][token=64]`. The generator packs `B.T` as the left matrix +operand and `A.T` as the right operand, so there is no runtime transpose. +Bias is one FP32 value per vocabulary entry, broadcast over tokens. +Supported K: 128, 256, 512, TE=16, VLEN=512. + +The kernel retains the packed FP16 1x4 matmul shape and four accumulator +tiles. One e32,m4 load provides four K-pair vectors; A is consumed by all +four tiles before replacement. B alternates v16-v19 and v24-v27, with the +next tile's load issued ahead of the current tile's MAC quartet. The final +MAC group completes before two-row VTMV drains. Bias is added to completed +FP32 logits before scratch stores (epilogue addition, not accumulator seed). + +A 1x4 block contains only 16 of the 64 vocabulary rows. All blocks must +finish before column softmax can normalize. C serves as in-place L1 +scratch for logits and exponentials, followed by final normalized stores. +This implementation does not claim to eliminate intermediate VSE stores. +No extra logits array is allocated. One L1 operand buffer is used. + +Softmax follows the baseline feature-major algorithm: each vector lane +owns one token; `vfmax.vv` and `vfadd.vv` accumulate across vocabulary +rows. No `vfred*` instructions are used. Shared `apps/include/ope_transformer.h` macros provide the math. +Exp uses the baseline clamped +Taylor polynomial and six squarings; reciprocal uses three Newton steps. +The -16 clamp intentionally matches baseline and approximates tiny tails. + +Register schedule (e32,m4, VL=64): + +- v0: column maximum, then reciprocal; v4: column sum. +- v8/v12: first pair of rows; v24/v28: alternate pair. +- v16/v20: independent polynomial temporaries. + +The next pair is loaded before computing the current pair. Polynomial +instructions alternate between the two rows to space dependent operations. +The final pair is drained without an out-of-bounds prefetch. Normalization +alternates v8/v12 loads, multiplies, and final stores. Actual overlap and +performance remain dependent on the RTL scoreboard and functional units. + +Targets: `test-vme-fp16-gemm-softmax-te16-rolled_M64_N64_K{128,256,512}`. +The reference uses PyTorch FP32 GEMM on quantized FP16 operands, broadcast +bias and true softmax, then transpose. Validation checks every output with +`2e-5 + 1e-3*abs(reference)` tolerance. + +The public math macros are also documented in `../../include/README.md`. ReLU and rsqrt are +provided for later applications; only exp/reciprocal are exercised here. + +## RTL validation + +Questa RTL, M=N=64, one core, one L1 buffer: + +| K | Cycles | Errors | Maximum absolute error | +|---|---:|---:|---:| +| 128 | 19,385 | 0 | 7.2122e-6 | +| 256 | 24,063 | 0 | 8.0466e-6 | +| 512 | 33,407 | 0 | 8.7619e-6 | + +Logs: `hw/system/spatz_cluster/logs/gemm-softmax/k{128,256,512}.log` +from the repository root. Cycles cover the kernel (GEMM, bias, scratch +passes, softmax), excluding initial DMA and golden-reference validation. +The kernel disassembly contains no horizontal reductions or floating-point +divide/sqrt/reciprocal-estimate instructions. The linked printf library may +contain unused floating-format paths; the benchmark prints only integers +and FP32 bit patterns. diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-softmax/data/layer.h similarity index 97% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/data/layer.h rename to sw/vme/apps/tew32/fp16-gemm-softmax/data/layer.h index 8d340d758..149698826 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/data/layer.h +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/data/layer.h @@ -27,4 +27,4 @@ typedef struct { uint32_t M; uint32_t N; uint32_t K; -} vme_matmul_layer; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/gemm.c b/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/gemm.c new file mode 100644 index 000000000..507bd6aa0 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/gemm.c @@ -0,0 +1,137 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "gemm.h" +#include "softmax.h" + +// Four packed K pairs per operand group; each VTFMM consumes two FP16 K values. +#define GEMM_CONFIG_ASM \ + "msetmtype %[mt], %[vt]\nmsettn zero, %[tn]\n" +#define GEMM_MAC4_ASM(tile, b0, b1, b2, b3) \ + "vtfmm.tvv " tile ", v0, " b0 "\n" \ + "vtfmm.tvv " tile ", v1, " b1 "\n" \ + "vtfmm.tvv " tile ", v2, " b2 "\n" \ + "vtfmm.tvv " tile ", v3, " b3 "\n" + +__attribute__((noinline, aligned(64))) void gemm_softmax_fp16_fp32( + const __fp16 *Atpack, const __fp16 *Bpack, const float *bias, float *C) +{ + const uintptr_t c_stride = 64 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t vtype = 0xC8; // e16, m1, tk=2 widening SDOTP + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + + // Compute each 16-vocab-row block with a 1x4 accumulator layout. + // The complete [vocab][token] scratch feeds one global column softmax. + for (uint32_t nb = 0; nb < 4; nb += 1) { + const uint32_t mb0 = nb * TE; + { + // Initialize and compute one complete 16-row output block. + uintptr_t a0p = (uintptr_t)(Atpack + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(Bpack + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(Bpack + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile( + "msetmtype %[mt], %[vt]\n" + "msettn zero, %[tn]\n" + "vtzero mt0\nvtzero mt4\nvtzero mt8\nvtzero mt12\n" + "li t0, %[groups]\n" + "1:\n" + "vsetvli zero, %[vl], e32, m4, ta, ma\n" + "vle32.v v0, (%[a])\naddi %[a], %[a], 256\n" + "vle32.v v16, (%[b0])\naddi %[b0], %[b0], 256\n" + "vsetvli zero, %[vl], e32, m4, ta, ma\n" + "vle32.v v24, (%[b1])\naddi %[b1], %[b1], 256\n" + GEMM_CONFIG_ASM + GEMM_MAC4_ASM("mt0", "v16", "v17", "v18", "v19") + "vsetvli zero, %[vl], e32, m4, ta, ma\n" + "vle32.v v16, (%[b2])\naddi %[b2], %[b2], 256\n" + GEMM_CONFIG_ASM + GEMM_MAC4_ASM("mt4", "v24", "v25", "v26", "v27") + "vsetvli zero, %[vl], e32, m4, ta, ma\n" + "vle32.v v24, (%[b3])\naddi %[b3], %[b3], 256\n" + GEMM_CONFIG_ASM + GEMM_MAC4_ASM("mt8", "v16", "v17", "v18", "v19") + GEMM_CONFIG_ASM + GEMM_MAC4_ASM("mt12", "v24", "v25", "v26", "v27") + "addi t0, t0, -1\nbnez t0, 1b\n" + : [a] "+&r"(a0p), [b0] "+&r"(b0p), [b1] "+&r"(b1p), + [b2] "+&r"(b2p), [b3] "+&r"(b3p) + : [mt] "r"(mtype), [vt] "r"(vtype), [tn] "r"(tile_vl), + [vl] "r"(group_vl), [groups] "i"(K_CHUNK / 8) + : "t0", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", + "v24", "v25", "v26", "v27", "memory"); + + /* Drain two complete feature-major rows at a time. */ + for (uint32_t row = 0; row < TE; row += 2) { + { + asm volatile( + "msetmtype %4, %5\n" + "msettn x0, %6\n" + "vtmv.v.t v8, %0\naddi %0, %0, 1\n" + "vtmv.v.t v12, %0\naddi %0, %0, 1\n" + "vtmv.v.t v9, %1\naddi %1, %1, 1\n" + "vtmv.v.t v13, %1\naddi %1, %1, 1\n" + "vtmv.v.t v10, %2\naddi %2, %2, 1\n" + "vtmv.v.t v14, %2\naddi %2, %2, 1\n" + "vtmv.v.t v11, %3\naddi %3, %3, 1\n" + "vtmv.v.t v15, %3\naddi %3, %3, 1\n" + : "+&r"(tss0), "+&r"(tss4), "+&r"(tss8), "+&r"(tss12) + : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) + : "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"); + } + // Feature-major logits: each lane is a token, each row a vocab entry. + // This is scratch storage; normalization spans ALL four 16-row blocks. + asm volatile( + "vsetvli x0, %[vl], e32, m4, ta, ma\n" + "vfadd.vf v8, v8, %[b0]\n" + "vfadd.vf v12, v12, %[b1]\n" + "vse32.v v8, (%[ptr])\n" + "addi %[ptr], %[ptr], %[stride]\n" + "vse32.v v12, (%[ptr])\n" + "addi %[ptr], %[ptr], %[stride]\n" + : [ptr] "+&r"(ptr) + : [vl] "r"(group_vl), [stride] "i"(c_stride), + [b0] "f"(bias[mb0 + row]), [b1] "f"(bias[mb0 + row + 1]) + : "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"); + } + } + } +#ifdef GEMM_SOFTMAX_DIAGNOSTIC + for (unsigned i = 0, count = 0; i < 4096; ++i) { + float d = C[i] - Logits_dram[i]; + if ((d > 0.001f || d < -0.001f) && count++ < 8) { + union { float f; unsigned u; } a = {C[i]}, b = {Logits_dram[i]}; + printf("Logits [%u,%u] got=%08x ref=%08x\n", i/64, i%64, a.u, b.u); + } + } +#endif + softmax_cols(C); +} diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/gemm.h b/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/gemm.h new file mode 100644 index 000000000..61a86c487 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/gemm.h @@ -0,0 +1,54 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// Left/right operands are packed B^T/A^T. K counts FP16 elements. +// bias[vocab] is broadcast over tokens; C[vocab][token] is FP32 softmax. +// Fixed M=N=64, TE=16, 1x4 accumulator layout, complete K panel. +void gemm_softmax_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, const float *bias, float *C); diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/softmax.h b/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/softmax.h new file mode 100644 index 000000000..04053c7fa --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/kernel/softmax.h @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once +#include "ope_softmax.h" diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/main.c b/sw/vme/apps/tew32/fp16-gemm-softmax/main.c new file mode 100644 index 000000000..c2dccfe95 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/main.c @@ -0,0 +1,241 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "gemm.h" +#include "kernel/gemm.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES + 64 * sizeof(float) + 64 <= SNRT_TCDM_SIZE, + "gemm buffers and bias exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + if (matmul_l.M != 64 || matmul_l.N != 64 || matmul_l.K != K_CHUNK) { + printf("softmax requires M=N=64 and a complete K panel\n"); + return 2; + } + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + float *const bias = (float *)snrt_l1alloc(64 * sizeof(float)); + if (!bias) return 2; + snrt_dma_start_1d(bias, Bias_dram, 64 * sizeof(float)); + snrt_dma_wait_all(); + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + gemm_softmax_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), bias, (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.00002f + 0.001f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (errors < 8) printf("\nMismatch [%u,%u] actual=0x%08x expected=0x%08x\n", row, col, float_bits(actual), float_bits(expected)); + ++row_errors; + ++errors; + } + } + + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 gemm-softmax %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-softmax/script/gen_data.py new file mode 100644 index 000000000..088f02144 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/script/gen_data.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 8 or K % 8: + parser.error("packed store-group kernel requires real K >= 8 and divisible by 8") + + if M != 64 or N != 64: + parser.error("1x4 column-softmax kernel currently requires M=N=64") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + bias = torch.randn((N,), dtype=torch.float32) + logits = A.to(torch.float32) @ B.to(torch.float32) + bias + C = torch.softmax(logits, dim=1).T.contiguous().numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(B.T.copy(), A.T.copy()) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "#ifdef GEMM_SOFTMAX_DIAGNOSTIC", + f'static float Logits_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(logits.T.contiguous().numpy()) + ";", + "#endif", + "// Output [vocab][token] = softmax(A*B + bias, dim=1).T.", + f'static float Bias_dram[{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(bias.numpy()) + ";", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// Packed left operand B^T; vocab rows. High halfword is even K.", + "// One e32,m4 load transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// Packed right operand A^T; token columns. Same halfword order.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_128.json new file mode 100644 index 000000000..fd14beb30 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_256.json new file mode 100644 index 000000000..b1b3633fc --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_512.json new file mode 100644 index 000000000..e130c061a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-softmax/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/CMakeLists.txt b/sw/vme/apps/tew32/fp16-gemm-unit/CMakeLists.txt new file mode 100644 index 000000000..862352b44 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/CMakeLists.txt @@ -0,0 +1,58 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +add_subdirectory(rolled rolled) + +set(_source "${CMAKE_CURRENT_SOURCE_DIR}") + +function(add_fp16_simple_gemm_unrolled variant) + foreach(_k IN ITEMS 128 256 512) + # Packing two FP16 values into one word preserves the original byte size. + math(EXPR _a_bytes "64 * ${_k} * 2") + math(EXPR _b_bytes "${_k} * 64 * 2") + math(EXPR _c_bytes "64 * 64 * 4") + math(EXPR _buffer_bytes + "${_a_bytes} + ${_b_bytes} + ${_c_bytes}") + math(EXPR _required_tcdm_bytes "2 * ${_buffer_bytes}") + if(_required_tcdm_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping fp16-gemm-unit unrolled ${variant} M64 N64 K${_k}: " + "double buffering requires ${_required_tcdm_bytes} TCDM bytes, " + "but only ${SNRT_TCDM_SIZE} are configured") + continue() + endif() + + set(_dir "${_source}/unrolled/${variant}/K${_k}") + set(_target "fp16-gemm-unit-unrolled-${variant}_M64_N64_K${_k}") + set(_config "${_dir}/script/matmul_64_64_${_k}.json") + set(_header "${_dir}/data/data_64_64_${_k}.h") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${_dir}/script/gen_data.py" + -c "${_config}" + DEPENDS "${_dir}/script/gen_data.py" + "${_config}" + "${_dir}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${_dir}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${_dir}" "${_dir}/kernel" "${_dir}/data") + target_compile_definitions(${_test_target} PRIVATE + MATMUL_K_CHUNK=${_k} + MATMUL_BUFFER_COUNT=2 + DATAHEADER="data/data_64_64_${_k}.h") + endif() + endforeach() +endfunction() + +# add_fp16_simple_gemm_unrolled(store_afterK) +# add_fp16_simple_gemm_unrolled(store_interleave) +# add_fp16_simple_gemm_unrolled(vsetvl) diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/CMakeLists.txt b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/CMakeLists.txt new file mode 100644 index 000000000..199897895 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/CMakeLists.txt @@ -0,0 +1,47 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +set(_source "${CMAKE_CURRENT_SOURCE_DIR}") + +foreach(_k IN ITEMS 128 256 512) + # Packing two FP16 values into one word preserves the original byte size. + math(EXPR _a_bytes "64 * ${_k} * 2") + math(EXPR _b_bytes "${_k} * 64 * 2") + math(EXPR _c_bytes "64 * 64 * 4") + math(EXPR _required_tcdm_bytes + "${_a_bytes} + ${_b_bytes} + ${_c_bytes}") + if(_required_tcdm_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping fp16-gemm-unit M64 N64 K${_k}: requires " + "${_required_tcdm_bytes} TCDM bytes, but only " + "${SNRT_TCDM_SIZE} are configured") + continue() + endif() + + set(_target "fp16-gemm-unit_M64_N64_K${_k}") + set(_config "${_source}/script/gemm_64_64_${_k}.json") + set(_header "${_source}/data/data_64_64_${_k}.h") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${_source}/script/gen_data.py" + -c "${_config}" + DEPENDS "${_source}/script/gen_data.py" + "${_config}" + "${_source}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${_source}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${_source}" "${_source}/kernel" "${_source}/data") + target_compile_definitions(${_test_target} PRIVATE + K_CHUNK=${_k} + DATAHEADER="data/data_64_64_${_k}.h") + endif() +endforeach() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/data/layer.h new file mode 100644 index 000000000..9b0bfbdef --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME gemm layer configuration. + * + * Describes a Cresult = A * B + Caddend problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} gemm_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/kernel/gemm.c b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/kernel/gemm.c new file mode 100644 index 000000000..6d42b551a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/kernel/gemm.c @@ -0,0 +1,1018 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include "gemm.h" +// FP16-to-FP32 GEMM with interleaved bias loads; variants with and without cross-block overlap. + +__attribute__((noinline, aligned(64))) void gemm_fp16_fp32_unit_vtle_no_crossblock(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + // Full 32x32 blocks; K is the real FP16 extent, >=24 and divisible by 8. + for (uint32_t mb = 0; mb < M; mb += BLOCK_DIM) { + for (uint32_t nb = 0; nb < N; nb += BLOCK_DIM) { + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)(Atpack + mb * K); + uintptr_t a1p = (uintptr_t)(Atpack + mb * K + tile_stride); + uintptr_t b0p = (uintptr_t)(Bpack + nb * K); + uintptr_t b1p = (uintptr_t)(Bpack + nb * K + tile_stride); + uintptr_t tss; + const uintptr_t c_tile_bytes = sizeof(float) * TE; + uintptr_t c00 = (uintptr_t)(C + mb * N + nb); + uintptr_t c01 = c00 + c_tile_bytes; + uintptr_t c10 = (uintptr_t)(C + (mb + TE) * N + nb); + uintptr_t c11 = (uintptr_t)(C + (mb + TE) * N + nb + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = N * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + const uintptr_t rewind_b0_bytes = 2 * operand_group_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + + asm volatile( + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + // Bootstrap mt0; preload the other tiles during preceding-tile MACs. + "li %[tss], 0\n" + "li %[loop], 16\n" + ".Lbias_mt0_%=:\n" + "vtle32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "addi %[tss], %[tss], 1\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lbias_mt0_%=\n" + "slli %[vl], %[c_stride], 4\n" + "sub %[c00], %[c00], %[vl]\n" + + // K group 0: interleave four bias rows after each preceding-tile MAC. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" + "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Load mt4 bias while computing mt0. + "li %[tss], 0x20000000\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "slli %[loop], %[c_stride], 4\n" + "sub %[c01], %[c01], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Load mt8 bias while computing mt4. + "li %[tss], 0x40000000\n" + "vtfmm.tvv mt4, v0, v24\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "slli %[loop], %[c_stride], 4\n" + "sub %[c10], %[c10], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Load mt12 bias while computing mt8. + "li %[tss], 0x60000000\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "slli %[loop], %[c_stride], 4\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" + "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // Middle K groups: four consecutive VTFMMs per load/compute phase. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" + "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtfmm.tvv mt0, v8, v24\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" + // Finish this block: compact 16-row mt8 store loop. + "7:\n" + "li %[loop], 16\n" + "2:\n" + "vtse32 %[tss], (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[tss], %[tss], 1\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 2b\n" + "8:\n" + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [rewind_b0_bytes] "r"(rewind_b0_bytes), + [operand_group_bytes] "r"(operand_group_bytes), + [middle_k_groups] "r"(middle_k_groups) + : "memory"); + } + } +} + +__attribute__((noinline, aligned(64))) void gemm_fp16_fp32_unit_crossblock(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + // Full 32x32 blocks; K is the real FP16 extent, >=24 and divisible by 8. + const uint32_t col_blocks = N / BLOCK_DIM; + const uint32_t row_blocks = M / BLOCK_DIM; + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t c_tile_bytes = sizeof(float) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + c_tile_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * N); + uintptr_t c11 = (uintptr_t)(C + TE * N + TE); + uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; + uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = N * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + const uintptr_t rewind_b0_bytes = 2 * operand_group_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + + asm volatile( + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + // Bootstrap mt0; preload the other tiles during preceding-tile MACs. + "li %[tss], 0\n" + "li %[loop], 16\n" + ".Lbias_mt0_%=:\n" + "vtle32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "addi %[tss], %[tss], 1\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lbias_mt0_%=\n" + "slli %[vl], %[c_stride], 4\n" + "sub %[c00], %[c00], %[vl]\n" + + // K group 0: interleave four bias rows after each preceding-tile MAC. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" + "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Load mt4 bias while computing mt0. + "li %[tss], 0x20000000\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "slli %[loop], %[c_stride], 4\n" + "sub %[c01], %[c01], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Load mt8 bias while computing mt4. + "li %[tss], 0x40000000\n" + "vtfmm.tvv mt4, v0, v24\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "slli %[loop], %[c_stride], 4\n" + "sub %[c10], %[c10], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Load mt12 bias while computing mt8. + "li %[tss], 0x60000000\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtle32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "slli %[loop], %[c_stride], 4\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" + "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // Middle K groups: four consecutive VTFMMs per load/compute phase. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" + "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtfmm.tvv mt0, v8, v24\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Defer mt8 to the next block's compute/store overlap. The final + // block branches to the compact drain loop below. + "lui %[tss], 0x40000\n" + "addi %[blocks], %[blocks], -1\n" + "beqz %[blocks], 7f\n" + + // Advance horizontally until the N panel is exhausted. At the row + // boundary, advance A by two tiles and rewind B to its first pair. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "j 5f\n" + "4:\n" + "sub %[cols], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[cols]\n" + "add %[a1p], %[a1p], %[cols]\n" + "slli %[loop], %[col_blocks], 1\n" + "addi %[loop], %[loop], -1\n" + "6:\n" + "sub %[b0p], %[b0p], %[cols]\n" + "sub %[b1p], %[b1p], %[cols]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 6b\n" + "mv %[cols], %[col_blocks]\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + + "5:\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // Cross-Block Compute-Store overlapped (no control-flow path) + // vtfmm mt0 + "lui %[tss], 0x40000\n" // mt8 + // Next block mt0: its previous result has already been stored. + "li %[bias_tss], 0x0\n" + "sub %[bias_ptr], %[c00], %[loop]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v0, v16\n" + // Next block mt4: its previous result has already been stored. + "li %[bias_tss], 0x20000000\n" + "sub %[bias_ptr], %[c01], %[loop]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v0, v24\n" + // Next block mt12: its previous result has already been stored. + "li %[bias_tss], 0x60000000\n" + "sub %[bias_ptr], %[c11], %[loop]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + // Next block mt8: its previous result has already been stored. + "li %[bias_tss], 0x40000000\n" + "sub %[bias_ptr], %[c10], %[loop]\n" + + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // vtfmm mt8 + + "vtfmm.tvv mt8, v8, v16\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "vtfmm.tvv mt8, v9, v17\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "j 9b\n" + + // Final block: compact 16-row mt8 store loop. + "7:\n" + "li %[loop], 16\n" + "2:\n" + "vtse32 %[tss], (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[tss], %[tss], 1\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 2b\n" + "8:\n" + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [tss] "=&r"(tss), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + [blocks] "+&r"(block_counter), + [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [rewind_b0_bytes] "r"(rewind_b0_bytes), + [operand_group_bytes] "r"(operand_group_bytes), + [middle_k_groups] "r"(middle_k_groups), + [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} + +__attribute__((noinline, aligned(64))) void gemm_fp16_fp32_unit_vle( + const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + const uintptr_t c_stride = 128 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t vtype = 0xC8; // e16, m1, tk=2 widening SDOTP + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + const uintptr_t bias_vl = 8 * TE; // m8: two contiguous 64-element FP32 rows + const uintptr_t operand_vector_bytes = TE * sizeof(uint32_t); + const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + + // Process each 16-row output block independently; no state crosses blocks. + for (uint32_t nb = 0; nb < 4; nb += 1) { + const uint32_t mb0 = nb * TE; + { + // Initialize and compute one complete 16-row output block. + uintptr_t a0p = (uintptr_t)(Atpack + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(Bpack + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(Bpack + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t bias_ptr = ptr; + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtzero mt0" : : : "memory"); + asm volatile("vtzero mt4" : : : "memory"); + asm volatile("vtzero mt8" : : : "memory"); + asm volatile("vtzero mt12" : : : "memory"); + + // Use the already-advanced A pointer as the loop induction variable. + const uintptr_t a_middle_end = a0p + (K_CHUNK / 8 - 1) * operand_group_bytes; + while (a0p != a_middle_end) { + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vle32.v v24, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v24", "memory"); + asm volatile("vle32.v v25, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v25", "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vle32.v v26, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v26", "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("vle32.v v27, (%0)\naddi %0, %0, %1" : "+r"(b1p) : "i"(operand_vector_bytes) : "v27", "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + // Keep the pointer comparison; otherwise Clang adds a separate countdown. + asm volatile("" : "+r"(a0p)); + } + + // Final K group: move each completed tile row to the vector registers. + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b1p) : "v24", "v25", "v26", "v27", "memory"); b1p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("vle32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfadd.vv v8, v8, v0" : : : "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"); + asm volatile("vfadd.vv v16, v16, v24" : : : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory"); + asm volatile("vse32.v v8, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vse32.v v16, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + + // Rows 4-15 + for (uint32_t row = 4; row < TE; row += 4) { + asm volatile("vle32.v v0, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfadd.vv v8, v8, v0" : : : "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "memory"); + asm volatile("vle32.v v24, (%0)\n" "addi %0, %0, %1" : "+r"(bias_ptr) : "i"(c_stride) : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vfadd.vv v16, v16, v24" : : : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory"); + asm volatile("vse32.v v8, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vse32.v v16, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + } + } + } +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/kernel/gemm.h similarity index 67% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/kernel/matmul.h rename to sw/vme/apps/tew32/fp16-gemm-unit/rolled/kernel/gemm.h index 099df1f0a..6dbec2448 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/kernel/gemm.h @@ -23,10 +23,11 @@ enum { CE = 8, TE = 16, - CHUNK_SIZE = 64, + CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, }; -// FP16 VME matmul over TE=16 with architectural tk=2. C is a padded -// M-by-CHUNK_SIZE L1 panel; M and N must not exceed 64. -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +void gemm_fp16_fp32_no_crossblock(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void gemm_fp16_fp32_crossblock(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void gemm_fp16_fp32_unit_vle(const __fp16 *Atpack, const __fp16 *Bpack, + float *C); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/main.c new file mode 100644 index 000000000..64c14f8d6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/main.c @@ -0,0 +1,101 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "gemm.h" +#include DATAHEADER +#include "kernel/gemm.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle) : : "memory"); + return cycle; +} + +static inline uint32_t float_bits(float value) { + union { float f; uint32_t u; } bits = {.f = value}; + return bits.u; +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + if (gemm_l.M != 64 || gemm_l.N != 64 || + (gemm_l.K != 128 && gemm_l.K != 256 && gemm_l.K != 512)) { + printf("Unsupported GEMM shape: expected M64_N64_K128, K256, or K512\n"); + return 2; + } + enum { + A_BYTES = sizeof(Atpack_dram), + B_BYTES = sizeof(Bpack_dram), + C_BYTES = sizeof(Caddend_dram), + BUFFER_BYTES = A_BYTES + B_BYTES + C_BYTES + }; + _Static_assert(BUFFER_BYTES <= SNRT_TCDM_SIZE, "GEMM buffers exceed TCDM"); + uint8_t *const buffer = (uint8_t *)snrt_l1alloc(BUFFER_BYTES); + if (buffer == 0) + return 2; + __fp16 *const Atpack = (__fp16 *)buffer; + __fp16 *const Bpack = (__fp16 *)(buffer + A_BYTES); + float *const C = (float *)(buffer + A_BYTES + B_BYTES); + + snrt_dma_start_1d(Atpack, Atpack_dram, A_BYTES); + snrt_dma_start_1d(Bpack, Bpack_dram, B_BYTES); + snrt_dma_start_1d(C, Caddend_dram, C_BYTES); + snrt_dma_wait_all(); + + const uint32_t start = get_cycle(); + gemm_fp16_fp32_unit_vle(Atpack, Bpack, C); + const uint32_t cycles = get_cycle() - start; + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < gemm_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < gemm_l.N; ++col) { + const uint32_t index = row * gemm_l.N + col; + const float expected = Cref_dram[index]; + const float delta = C[index] > expected ? C[index] - expected : expected - C[index]; + const float tolerance = 0.001f + 0.001f * (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols %u", row, col); + else + printf(",%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + const uint64_t useful_macs = (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; + const uint64_t peak_macs = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization = peak_macs ? useful_macs * 10000 / peak_macs : 0; + printf("\n=== TEW32 FP16 unit-scaled GEMM M%u_N%u_K%u (TE=16, 1 core) ===\n", + gemm_l.M, gemm_l.N, gemm_l.K); + printf("Cycles: %u\nUtilization: %u.%02u%%\n", cycles, utilization / 100, utilization % 100); + printf("MaxAbsErr: 0x%08x\nErrors: %u\nStatus: %s\n", + float_bits(max_abs), errors, errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_128.json new file mode 100644 index 000000000..f35ec0e9d --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_128.json @@ -0,0 +1,9 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_256.json new file mode 100644 index 000000000..fec92ef06 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_256.json @@ -0,0 +1,9 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_512.json new file mode 100644 index 000000000..266175b16 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gemm_64_64_512.json @@ -0,0 +1,9 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gen_data.py new file mode 100644 index 000000000..6b659fee9 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/gen_data.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin, ETH Zürich +# +# +# Usage: +# python3 script/gen_data.py -c script/gemm_64_64_64.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Required fields: TM, TN; optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a, b, te): + """Word high half = even K, low half = odd K; matches FP16 VTSE packing.""" + m, k = a.shape + n = b.shape[1] + abits = a.view(np.uint16).astype(np.uint32) + bbits = b.view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate packed FP16-to-FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + TM = int(param["TM"]) + TN = int(param["TN"]) + seed = int(param.get("seed", 42)) + + if M != 64 or N != 64 or K not in (128, 256, 512) or (TM, TN) != (16, 16): + parser.error("this test targets M64_N64_K128, K256, or K512 with TE=16") + + torch.manual_seed(seed) + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16).numpy() + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16).numpy() + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + # Quantize the inputs first, then accumulate each K pair into FP32. + C = Caddend.copy() + for k in range(0, K, 2): + pair = (A[:, k, None].astype(np.float64) * B[None, k, :].astype(np.float64) + + A[:, k+1, None].astype(np.float64) * B[None, k+1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + A_pack, B_pack = pack_pairs(A, B, TM) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin, ETH Zürich", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + "", + "const gemm_layer gemm_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 golden reference: Cresult = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// FP32 initial C, with an independent addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "// Atpack[m_tile][K/2][TE]: high half even K, low half odd K.", + f'static uint32_t Atpack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// Bpack[n_tile][K/2][TE], matching Atpack halfword order.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/matmul_64_64_64.json new file mode 100644 index 000000000..2206091e2 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/rolled/script/matmul_64_64_64.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 128, + N: 128, + K: 128, + TM: 32, + TN: 32, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/data/layer.h similarity index 97% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/data/layer.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/data/layer.h index 8d340d758..149698826 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/data/layer.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/data/layer.h @@ -27,4 +27,4 @@ typedef struct { uint32_t M; uint32_t N; uint32_t K; -} vme_matmul_layer; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/kernel/matmul.c new file mode 100644 index 000000000..408e7dec3 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/kernel/matmul.c @@ -0,0 +1,3357 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/kernel/matmul.h similarity index 76% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/kernel/matmul.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/kernel/matmul.h index 6a6ad3374..e635de107 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/kernel/matmul.h @@ -23,10 +23,9 @@ enum { CE = 8, TE = 16, - CHUNK_SIZE = 64, + CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, }; -// FP32 matmul over TE=16 and a 256-element K chunk. C is a padded -// M-by-CHUNK_SIZE L1 panel; M and N must not exceed 64. -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/main.c similarity index 60% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/main.c rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/main.c index 2c29ce22b..da2ca42bb 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/main.c +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/main.c @@ -16,22 +16,27 @@ // // Author: Pei-Yu Lin, ETH Zürich -// main.c - fp32 x fp32 = fp32 VME GEMM functional check - #include -#include #include +#include "matmul.h" #include DATAHEADER #include "kernel/matmul.c" +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + enum { - TCDM_BYTES = 256 * 1024, M_CHUNK = CHUNK_SIZE, N_CHUNK = CHUNK_SIZE, - K_CHUNK = CHUNK_SIZE, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), A_TILES = M_CHUNK / TE, B_TILES = N_CHUNK / TE, A_BYTES = A_TILES * TILE_BYTES, @@ -42,22 +47,23 @@ enum { B_OFFSET = A_OFFSET + A_BYTES, C_OFFSET = B_OFFSET + B_BYTES, BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = 2, // 2 - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = L1_BUFFER_STRIDE + BUFFER_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, - "one M64xN64xK64 TEW32 buffer must fit in 128 KiB"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "two matmul buffers exceed the 256 KiB TCDM"); + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); -static const float zero_tile_dram[TILE_INPUTS] +static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; -static uint8_t *l1_base; -static float *partials; -static float *c_out; +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; static inline uint32_t min_u32(uint32_t a, uint32_t b) { return a < b ? a : b; @@ -77,12 +83,7 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static inline uint32_t cycle_count(void) { - uint32_t cycles; - asm volatile("csrr %0, mcycle" : "=r"(cycles)); - return cycles; -} -static snrt_dma_txid_t fill_panel(float *dst, const float *src, +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, uint32_t valid_k, @@ -97,16 +98,13 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, if (valid_tiles == 0 || valid_k == 0) return clear_tid; - const size_t copy_bytes = valid_k * TE * sizeof(float); - const size_t src_stride = src_tile_k * TE * sizeof(float); - // The assembly kernel advances between packed tiles by 16 * valid_k - // elements. Keep the L1 panel compact even though each buffer reserves - // space for the maximum K_CHUNK. + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { const uint32_t first_tile = mb * A_TILES; const uint32_t tile_count = (matmul_l.M + TE - 1) / TE; @@ -116,14 +114,15 @@ static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { const uint32_t first_tile = nb * B_TILES; const uint32_t tile_count = (matmul_l.N + TE - 1) / TE; @@ -133,53 +132,74 @@ static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); } -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - return (float)partial[(size_t)row * N_CHUNK + col]; +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); } int main(void) { if (snrt_cluster_core_idx() != 0) return 0; + // This unrolled kernel executes exactly 128 original FP16 K elements. + _Static_assert(K_CHUNK == 128, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; const uint32_t jobs = mb_count * nb_count; const uint32_t iterations = jobs * kb_count; - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); - c_out = (float *)snrt_l3alloc((size_t)matmul_l.M * matmul_l.N * - sizeof(float)); - if (l1_base == 0 || partials == 0 || c_out == 0) { - printf("fp16 matmul allocation failure\n"); + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); return 2; } - snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0, 0}; + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; uint32_t current_buffer = 0; - input_tid[0] = fill_buffer(0, 0, 0, 0); + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); snrt_dma_wait(input_tid[0]); - if (iterations > 1) { + if (L1_BUFFER_COUNT == 2 && iterations > 1) { const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); } uint32_t start_cycle = 0; @@ -193,26 +213,34 @@ int main(void) { const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; - uint32_t fcsr; if (iter == 0) - start_cycle = cycle_count(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); - end_cycle = cycle_count(); + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); - snrt_dma_start_1d(partials + (size_t)iter * C_ELEMENTS, - base + C_OFFSET, C_BYTES); + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); const uint32_t next_iter = iter + 1; - if (next_iter < iterations) { + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { const uint32_t next_buffer = current_buffer ^ 1; snrt_dma_wait(input_tid[next_buffer]); const uint32_t refill_iter = iter + 2; if (refill_iter < iterations) { const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, + l1_base, current_buffer, refill_job / nb_count, refill_job % nb_count, refill_iter % kb_count); } current_buffer = next_buffer; @@ -220,7 +248,10 @@ int main(void) { } snrt_dma_wait_all(); + uint32_t errors = 0; + float max_abs = 0.0f; for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; for (uint32_t col = 0; col < matmul_l.N; ++col) { const uint32_t mb = row / M_CHUNK; const uint32_t nb = col / N_CHUNK; @@ -231,19 +262,10 @@ int main(void) { for (uint32_t kb = 0; kb < kb_count; ++kb) { const float *partial = partials + (size_t)(job * kb_count + kb) * C_ELEMENTS; - sum += partial_at(partial, local_row, local_col); + sum += partial[(size_t)local_row * N_CHUNK + local_col]; } - c_out[(size_t)row * matmul_l.N + col] = sum; - } - } - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t row = 0; row < matmul_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < matmul_l.N; ++col) { const size_t index = (size_t)row * matmul_l.N + col; - const float actual = c_out[index]; + const float actual = sum; const float expected = Cref_dram[index]; const float delta = actual > expected ? actual - expected : expected - actual; @@ -251,7 +273,7 @@ int main(void) { (expected < 0.0f ? -expected : expected); if (delta > max_abs) max_abs = delta; - if (delta > tolerance) { + if (!(delta <= tolerance)) { if (row_errors == 0) printf("row %u: errors at cols ", row); else @@ -268,13 +290,13 @@ int main(void) { const uint32_t cycles = end_cycle - start_cycle; const uint64_t useful_fmas = (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; - const uint64_t peak_fmas = (uint64_t)cycles * CE * CE; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; const uint32_t utilization_bp = peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - printf("\n=== TEW32 FP32 matmul %ux%ux%u " + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", matmul_l.M, matmul_l.N, matmul_l.K, M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/script/gen_data.py new file mode 100644 index 000000000..19b79e3e0 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("store-afterK requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/script/matmul_64_64_128.json new file mode 100644 index 000000000..3a9efc194 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K128/script/matmul_64_64_128.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/data/layer.h similarity index 97% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/data/layer.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/data/layer.h index 8d340d758..149698826 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/data/layer.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/data/layer.h @@ -27,4 +27,4 @@ typedef struct { uint32_t M; uint32_t N; uint32_t K; -} vme_matmul_layer; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/kernel/matmul.c new file mode 100644 index 000000000..f85169440 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/kernel/matmul.c @@ -0,0 +1,5980 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[loop], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/kernel/matmul.h similarity index 76% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/kernel/matmul.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/kernel/matmul.h index 099df1f0a..e635de107 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/kernel/matmul.h @@ -23,10 +23,9 @@ enum { CE = 8, TE = 16, - CHUNK_SIZE = 64, + CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, }; -// FP16 VME matmul over TE=16 with architectural tk=2. C is a padded -// M-by-CHUNK_SIZE L1 panel; M and N must not exceed 64. -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/main.c similarity index 60% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/main.c rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/main.c index 2c29ce22b..fda2c31d6 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/main.c +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/main.c @@ -16,22 +16,27 @@ // // Author: Pei-Yu Lin, ETH Zürich -// main.c - fp32 x fp32 = fp32 VME GEMM functional check - #include -#include #include +#include "matmul.h" #include DATAHEADER #include "kernel/matmul.c" +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + enum { - TCDM_BYTES = 256 * 1024, M_CHUNK = CHUNK_SIZE, N_CHUNK = CHUNK_SIZE, - K_CHUNK = CHUNK_SIZE, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), A_TILES = M_CHUNK / TE, B_TILES = N_CHUNK / TE, A_BYTES = A_TILES * TILE_BYTES, @@ -42,22 +47,23 @@ enum { B_OFFSET = A_OFFSET + A_BYTES, C_OFFSET = B_OFFSET + B_BYTES, BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = 2, // 2 - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = L1_BUFFER_STRIDE + BUFFER_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, - "one M64xN64xK64 TEW32 buffer must fit in 128 KiB"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "two matmul buffers exceed the 256 KiB TCDM"); + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); -static const float zero_tile_dram[TILE_INPUTS] +static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; -static uint8_t *l1_base; -static float *partials; -static float *c_out; +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; static inline uint32_t min_u32(uint32_t a, uint32_t b) { return a < b ? a : b; @@ -77,12 +83,7 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static inline uint32_t cycle_count(void) { - uint32_t cycles; - asm volatile("csrr %0, mcycle" : "=r"(cycles)); - return cycles; -} -static snrt_dma_txid_t fill_panel(float *dst, const float *src, +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, uint32_t valid_k, @@ -97,16 +98,13 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, if (valid_tiles == 0 || valid_k == 0) return clear_tid; - const size_t copy_bytes = valid_k * TE * sizeof(float); - const size_t src_stride = src_tile_k * TE * sizeof(float); - // The assembly kernel advances between packed tiles by 16 * valid_k - // elements. Keep the L1 panel compact even though each buffer reserves - // space for the maximum K_CHUNK. + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { const uint32_t first_tile = mb * A_TILES; const uint32_t tile_count = (matmul_l.M + TE - 1) / TE; @@ -116,14 +114,15 @@ static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { const uint32_t first_tile = nb * B_TILES; const uint32_t tile_count = (matmul_l.N + TE - 1) / TE; @@ -133,53 +132,74 @@ static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); } -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - return (float)partial[(size_t)row * N_CHUNK + col]; +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); } int main(void) { if (snrt_cluster_core_idx() != 0) return 0; + // This unrolled kernel executes exactly 256 original FP16 K elements. + _Static_assert(K_CHUNK == 256, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; const uint32_t jobs = mb_count * nb_count; const uint32_t iterations = jobs * kb_count; - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); - c_out = (float *)snrt_l3alloc((size_t)matmul_l.M * matmul_l.N * - sizeof(float)); - if (l1_base == 0 || partials == 0 || c_out == 0) { - printf("fp16 matmul allocation failure\n"); + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); return 2; } - snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0, 0}; + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; uint32_t current_buffer = 0; - input_tid[0] = fill_buffer(0, 0, 0, 0); + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); snrt_dma_wait(input_tid[0]); - if (iterations > 1) { + if (L1_BUFFER_COUNT == 2 && iterations > 1) { const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); } uint32_t start_cycle = 0; @@ -193,26 +213,34 @@ int main(void) { const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; - uint32_t fcsr; if (iter == 0) - start_cycle = cycle_count(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); - end_cycle = cycle_count(); + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); - snrt_dma_start_1d(partials + (size_t)iter * C_ELEMENTS, - base + C_OFFSET, C_BYTES); + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); const uint32_t next_iter = iter + 1; - if (next_iter < iterations) { + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { const uint32_t next_buffer = current_buffer ^ 1; snrt_dma_wait(input_tid[next_buffer]); const uint32_t refill_iter = iter + 2; if (refill_iter < iterations) { const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, + l1_base, current_buffer, refill_job / nb_count, refill_job % nb_count, refill_iter % kb_count); } current_buffer = next_buffer; @@ -220,7 +248,10 @@ int main(void) { } snrt_dma_wait_all(); + uint32_t errors = 0; + float max_abs = 0.0f; for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; for (uint32_t col = 0; col < matmul_l.N; ++col) { const uint32_t mb = row / M_CHUNK; const uint32_t nb = col / N_CHUNK; @@ -231,19 +262,10 @@ int main(void) { for (uint32_t kb = 0; kb < kb_count; ++kb) { const float *partial = partials + (size_t)(job * kb_count + kb) * C_ELEMENTS; - sum += partial_at(partial, local_row, local_col); + sum += partial[(size_t)local_row * N_CHUNK + local_col]; } - c_out[(size_t)row * matmul_l.N + col] = sum; - } - } - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t row = 0; row < matmul_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < matmul_l.N; ++col) { const size_t index = (size_t)row * matmul_l.N + col; - const float actual = c_out[index]; + const float actual = sum; const float expected = Cref_dram[index]; const float delta = actual > expected ? actual - expected : expected - actual; @@ -251,7 +273,7 @@ int main(void) { (expected < 0.0f ? -expected : expected); if (delta > max_abs) max_abs = delta; - if (delta > tolerance) { + if (!(delta <= tolerance)) { if (row_errors == 0) printf("row %u: errors at cols ", row); else @@ -268,13 +290,13 @@ int main(void) { const uint32_t cycles = end_cycle - start_cycle; const uint64_t useful_fmas = (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; - const uint64_t peak_fmas = (uint64_t)cycles * CE * CE; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; const uint32_t utilization_bp = peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - printf("\n=== TEW32 FP32 matmul %ux%ux%u " + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", matmul_l.M, matmul_l.N, matmul_l.K, M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/script/gen_data.py new file mode 100644 index 000000000..8fc3f9473 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_256.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_256.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("store-afterK requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/script/matmul_64_64_256.json new file mode 100644 index 000000000..db3d003b3 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K256/script/matmul_64_64_256.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/data/layer.h similarity index 97% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/data/layer.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/data/layer.h index 8d340d758..149698826 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/data/layer.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/data/layer.h @@ -27,4 +27,4 @@ typedef struct { uint32_t M; uint32_t N; uint32_t K; -} vme_matmul_layer; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/kernel/matmul.c new file mode 100644 index 000000000..4b35a2bbb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/kernel/matmul.c @@ -0,0 +1,11228 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[loop], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/kernel/matmul.h similarity index 76% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/kernel/matmul.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/kernel/matmul.h index 099df1f0a..e635de107 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/kernel/matmul.h @@ -23,10 +23,9 @@ enum { CE = 8, TE = 16, - CHUNK_SIZE = 64, + CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, }; -// FP16 VME matmul over TE=16 with architectural tk=2. C is a padded -// M-by-CHUNK_SIZE L1 panel; M and N must not exceed 64. -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/main.c similarity index 60% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/main.c rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/main.c index 596771f5b..4ba920fd4 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/main.c +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/main.c @@ -16,22 +16,27 @@ // // Author: Pei-Yu Lin, ETH Zürich -// main.c - fp32 x fp32 = fp32 VME GEMM functional check - #include -#include #include +#include "matmul.h" #include DATAHEADER #include "kernel/matmul.c" +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + enum { - TCDM_BYTES = 256 * 1024, M_CHUNK = CHUNK_SIZE, N_CHUNK = CHUNK_SIZE, - K_CHUNK = 2 * CHUNK_SIZE, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), A_TILES = M_CHUNK / TE, B_TILES = N_CHUNK / TE, A_BYTES = A_TILES * TILE_BYTES, @@ -42,22 +47,23 @@ enum { B_OFFSET = A_OFFSET + A_BYTES, C_OFFSET = B_OFFSET + B_BYTES, BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = 2, // 2 - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = L1_BUFFER_STRIDE + BUFFER_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, - "one M64xN64xK128 TEW32 buffer must fit in 128 KiB"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "two matmul buffers exceed the 256 KiB TCDM"); + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); -static const float zero_tile_dram[TILE_INPUTS] +static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; -static uint8_t *l1_base; -static float *partials; -static float *c_out; +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; static inline uint32_t min_u32(uint32_t a, uint32_t b) { return a < b ? a : b; @@ -77,12 +83,7 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static inline uint32_t cycle_count(void) { - uint32_t cycles; - asm volatile("csrr %0, mcycle" : "=r"(cycles)); - return cycles; -} -static snrt_dma_txid_t fill_panel(float *dst, const float *src, +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, uint32_t valid_k, @@ -97,16 +98,13 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, if (valid_tiles == 0 || valid_k == 0) return clear_tid; - const size_t copy_bytes = valid_k * TE * sizeof(float); - const size_t src_stride = src_tile_k * TE * sizeof(float); - // The assembly kernel advances between packed tiles by 16 * valid_k - // elements. Keep the L1 panel compact even though each buffer reserves - // space for the maximum K_CHUNK. + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { const uint32_t first_tile = mb * A_TILES; const uint32_t tile_count = (matmul_l.M + TE - 1) / TE; @@ -116,14 +114,15 @@ static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { const uint32_t first_tile = nb * B_TILES; const uint32_t tile_count = (matmul_l.N + TE - 1) / TE; @@ -133,53 +132,74 @@ static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); } -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - return (float)partial[(size_t)row * N_CHUNK + col]; +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); } int main(void) { if (snrt_cluster_core_idx() != 0) return 0; + // This unrolled kernel executes exactly 512 original FP16 K elements. + _Static_assert(K_CHUNK == 512, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; const uint32_t jobs = mb_count * nb_count; const uint32_t iterations = jobs * kb_count; - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); - c_out = (float *)snrt_l3alloc((size_t)matmul_l.M * matmul_l.N * - sizeof(float)); - if (l1_base == 0 || partials == 0 || c_out == 0) { - printf("fp16 matmul allocation failure\n"); + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); return 2; } - snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0, 0}; + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; uint32_t current_buffer = 0; - input_tid[0] = fill_buffer(0, 0, 0, 0); + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); snrt_dma_wait(input_tid[0]); - if (iterations > 1) { + if (L1_BUFFER_COUNT == 2 && iterations > 1) { const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); } uint32_t start_cycle = 0; @@ -193,26 +213,34 @@ int main(void) { const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; - uint32_t fcsr; if (iter == 0) - start_cycle = cycle_count(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); - end_cycle = cycle_count(); + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); - snrt_dma_start_1d(partials + (size_t)iter * C_ELEMENTS, - base + C_OFFSET, C_BYTES); + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); const uint32_t next_iter = iter + 1; - if (next_iter < iterations) { + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { const uint32_t next_buffer = current_buffer ^ 1; snrt_dma_wait(input_tid[next_buffer]); const uint32_t refill_iter = iter + 2; if (refill_iter < iterations) { const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, + l1_base, current_buffer, refill_job / nb_count, refill_job % nb_count, refill_iter % kb_count); } current_buffer = next_buffer; @@ -220,7 +248,10 @@ int main(void) { } snrt_dma_wait_all(); + uint32_t errors = 0; + float max_abs = 0.0f; for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; for (uint32_t col = 0; col < matmul_l.N; ++col) { const uint32_t mb = row / M_CHUNK; const uint32_t nb = col / N_CHUNK; @@ -231,19 +262,10 @@ int main(void) { for (uint32_t kb = 0; kb < kb_count; ++kb) { const float *partial = partials + (size_t)(job * kb_count + kb) * C_ELEMENTS; - sum += partial_at(partial, local_row, local_col); + sum += partial[(size_t)local_row * N_CHUNK + local_col]; } - c_out[(size_t)row * matmul_l.N + col] = sum; - } - } - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t row = 0; row < matmul_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < matmul_l.N; ++col) { const size_t index = (size_t)row * matmul_l.N + col; - const float actual = c_out[index]; + const float actual = sum; const float expected = Cref_dram[index]; const float delta = actual > expected ? actual - expected : expected - actual; @@ -251,7 +273,7 @@ int main(void) { (expected < 0.0f ? -expected : expected); if (delta > max_abs) max_abs = delta; - if (delta > tolerance) { + if (!(delta <= tolerance)) { if (row_errors == 0) printf("row %u: errors at cols ", row); else @@ -268,13 +290,13 @@ int main(void) { const uint32_t cycles = end_cycle - start_cycle; const uint64_t useful_fmas = (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; - const uint64_t peak_fmas = (uint64_t)cycles * CE * CE; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; const uint32_t utilization_bp = peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - printf("\n=== TEW32 FP32 matmul %ux%ux%u " + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", matmul_l.M, matmul_l.N, matmul_l.K, M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/script/gen_data.py new file mode 100644 index 000000000..85c6c99ad --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_512.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_512.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("store-afterK requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/script/matmul_64_64_512.json new file mode 100644 index 000000000..4a78f5d6f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_afterK/K512/script/matmul_64_64_512.json @@ -0,0 +1,11 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/kernel/matmul.c new file mode 100644 index 000000000..485454cde --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/kernel/matmul.c @@ -0,0 +1,3596 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Complete the block before changing any A/B/C panel pointer. The + // former cross-block path deferred this mt8 store while beginning the + // next block, but its FP32 register preload was incomplete. + + // // block transition + // "addi %[blocks], %[blocks], -1\n" + // "beqz %[blocks], 8f\n" + + // // Advance horizontally until the N panel is exhausted. At the row + // // boundary, advance A by two tiles and rewind B to its first pair. + // "addi %[cols], %[cols], -1\n" + // "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + // Flattened successor block. The previous mt8 is fully stored before + // this block reuses it, so TSS starts at mt0,row0. + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "j 9b\n" + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 2b\n" + // "8:\n" + + // Move from the top-right 32x32 block to the bottom-left block. + // A advances by one 32-row panel, B rewinds to its first column + // panel, and C advances diagonally from (0,32) to (32,0). + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // ".Lmatmul_block_%=:\n" + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + // Block 2 -> 3 moves from the top-right to the bottom-left panel. + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[blocks], %[blocks], -1\n" + // "beqz %[blocks], 8f\n" + + // // Advance horizontally until the N panel is exhausted. At the row + // // boundary, advance A by two tiles and rewind B to its first pair. + // "addi %[cols], %[cols], -1\n" + // "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + // "j 5f\n" + // "4:\n" + // "sub %[cols], %[b1p], %[b0p]\n" + // "add %[a0p], %[a0p], %[cols]\n" + // "add %[a1p], %[a1p], %[cols]\n" + // "slli %[loop], %[col_blocks], 1\n" + // "addi %[loop], %[loop], -1\n" + // "6:\n" + // "sub %[b0p], %[b0p], %[cols]\n" + // "sub %[b1p], %[b1p], %[cols]\n" + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 6b\n" + // "mv %[cols], %[col_blocks]\n" + // "slli %[loop], %[c_stride], 4\n" + // "addi %[loop], %[loop], -128\n" + // "sub %[loop], x0, %[loop]\n" + + // "5:\n" + // "sub %[c00], %[c00], %[loop]\n" + // "sub %[c01], %[c01], %[loop]\n" + // "sub %[c10], %[c10], %[loop]\n" + // "sub %[c11], %[c11], %[loop]\n" + // "j .Lmatmul_block_%=\n" + + // Flattened successor block. The previous mt8 is fully stored before + // this block reuses it, so TSS starts at mt0,row0. + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + // "j 9b\n" + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Final block: compact 16-row mt8 store loop. + // "7:\n" + // "li %[loop], 16\n" + // "2:\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 2b\n" + // "8:\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/kernel/matmul.h similarity index 76% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/kernel/matmul.h rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/kernel/matmul.h index 099df1f0a..e635de107 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/kernel/matmul.h @@ -23,10 +23,9 @@ enum { CE = 8, TE = 16, - CHUNK_SIZE = 64, + CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, }; -// FP16 VME matmul over TE=16 with architectural tk=2. C is a padded -// M-by-CHUNK_SIZE L1 panel; M and N must not exceed 64. -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/main.c similarity index 60% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/main.c rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/main.c index 2c29ce22b..da2ca42bb 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/main.c @@ -16,22 +16,27 @@ // // Author: Pei-Yu Lin, ETH Zürich -// main.c - fp32 x fp32 = fp32 VME GEMM functional check - #include -#include #include +#include "matmul.h" #include DATAHEADER #include "kernel/matmul.c" +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + enum { - TCDM_BYTES = 256 * 1024, M_CHUNK = CHUNK_SIZE, N_CHUNK = CHUNK_SIZE, - K_CHUNK = CHUNK_SIZE, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), A_TILES = M_CHUNK / TE, B_TILES = N_CHUNK / TE, A_BYTES = A_TILES * TILE_BYTES, @@ -42,22 +47,23 @@ enum { B_OFFSET = A_OFFSET + A_BYTES, C_OFFSET = B_OFFSET + B_BYTES, BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = 2, // 2 - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = L1_BUFFER_STRIDE + BUFFER_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, - "one M64xN64xK64 TEW32 buffer must fit in 128 KiB"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "two matmul buffers exceed the 256 KiB TCDM"); + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); -static const float zero_tile_dram[TILE_INPUTS] +static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; -static uint8_t *l1_base; -static float *partials; -static float *c_out; +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; static inline uint32_t min_u32(uint32_t a, uint32_t b) { return a < b ? a : b; @@ -77,12 +83,7 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static inline uint32_t cycle_count(void) { - uint32_t cycles; - asm volatile("csrr %0, mcycle" : "=r"(cycles)); - return cycles; -} -static snrt_dma_txid_t fill_panel(float *dst, const float *src, +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, uint32_t valid_k, @@ -97,16 +98,13 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, if (valid_tiles == 0 || valid_k == 0) return clear_tid; - const size_t copy_bytes = valid_k * TE * sizeof(float); - const size_t src_stride = src_tile_k * TE * sizeof(float); - // The assembly kernel advances between packed tiles by 16 * valid_k - // elements. Keep the L1 panel compact even though each buffer reserves - // space for the maximum K_CHUNK. + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { const uint32_t first_tile = mb * A_TILES; const uint32_t tile_count = (matmul_l.M + TE - 1) / TE; @@ -116,14 +114,15 @@ static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { const uint32_t first_tile = nb * B_TILES; const uint32_t tile_count = (matmul_l.N + TE - 1) / TE; @@ -133,53 +132,74 @@ static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 + const uint32_t *src = valid_tiles == 0 ? zero_tile_dram : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); } -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); } -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - return (float)partial[(size_t)row * N_CHUNK + col]; +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); } int main(void) { if (snrt_cluster_core_idx() != 0) return 0; + // This unrolled kernel executes exactly 128 original FP16 K elements. + _Static_assert(K_CHUNK == 128, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; const uint32_t jobs = mb_count * nb_count; const uint32_t iterations = jobs * kb_count; - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); - c_out = (float *)snrt_l3alloc((size_t)matmul_l.M * matmul_l.N * - sizeof(float)); - if (l1_base == 0 || partials == 0 || c_out == 0) { - printf("fp16 matmul allocation failure\n"); + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); return 2; } - snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0, 0}; + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; uint32_t current_buffer = 0; - input_tid[0] = fill_buffer(0, 0, 0, 0); + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); snrt_dma_wait(input_tid[0]); - if (iterations > 1) { + if (L1_BUFFER_COUNT == 2 && iterations > 1) { const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); } uint32_t start_cycle = 0; @@ -193,26 +213,34 @@ int main(void) { const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; - uint32_t fcsr; if (iter == 0) - start_cycle = cycle_count(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); - end_cycle = cycle_count(); + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); - snrt_dma_start_1d(partials + (size_t)iter * C_ELEMENTS, - base + C_OFFSET, C_BYTES); + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); const uint32_t next_iter = iter + 1; - if (next_iter < iterations) { + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { const uint32_t next_buffer = current_buffer ^ 1; snrt_dma_wait(input_tid[next_buffer]); const uint32_t refill_iter = iter + 2; if (refill_iter < iterations) { const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, + l1_base, current_buffer, refill_job / nb_count, refill_job % nb_count, refill_iter % kb_count); } current_buffer = next_buffer; @@ -220,7 +248,10 @@ int main(void) { } snrt_dma_wait_all(); + uint32_t errors = 0; + float max_abs = 0.0f; for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; for (uint32_t col = 0; col < matmul_l.N; ++col) { const uint32_t mb = row / M_CHUNK; const uint32_t nb = col / N_CHUNK; @@ -231,19 +262,10 @@ int main(void) { for (uint32_t kb = 0; kb < kb_count; ++kb) { const float *partial = partials + (size_t)(job * kb_count + kb) * C_ELEMENTS; - sum += partial_at(partial, local_row, local_col); + sum += partial[(size_t)local_row * N_CHUNK + local_col]; } - c_out[(size_t)row * matmul_l.N + col] = sum; - } - } - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t row = 0; row < matmul_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < matmul_l.N; ++col) { const size_t index = (size_t)row * matmul_l.N + col; - const float actual = c_out[index]; + const float actual = sum; const float expected = Cref_dram[index]; const float delta = actual > expected ? actual - expected : expected - actual; @@ -251,7 +273,7 @@ int main(void) { (expected < 0.0f ? -expected : expected); if (delta > max_abs) max_abs = delta; - if (delta > tolerance) { + if (!(delta <= tolerance)) { if (row_errors == 0) printf("row %u: errors at cols ", row); else @@ -268,13 +290,13 @@ int main(void) { const uint32_t cycles = end_cycle - start_cycle; const uint64_t useful_fmas = (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; - const uint64_t peak_fmas = (uint64_t)cycles * CE * CE; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; const uint32_t utilization_bp = peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - printf("\n=== TEW32 FP32 matmul %ux%ux%u " + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", matmul_l.M, matmul_l.N, matmul_l.K, M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/script/gen_data.py new file mode 100644 index 000000000..2096fc2bf --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/script/matmul_64_64_128.json new file mode 100644 index 000000000..3a9efc194 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K128/script/matmul_64_64_128.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/kernel/matmul.c new file mode 100644 index 000000000..c0749fc9c --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/kernel/matmul.c @@ -0,0 +1,6476 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Complete the block before changing any A/B/C panel pointer. The + // former cross-block path deferred this mt8 store while beginning the + // next block, but its FP32 register preload was incomplete. + + // // block transition + // "addi %[blocks], %[blocks], -1\n" + // "beqz %[blocks], 8f\n" + + // // Advance horizontally until the N panel is exhausted. At the row + // // boundary, advance A by two tiles and rewind B to its first pair. + // "addi %[cols], %[cols], -1\n" + // "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + // Flattened successor block. The previous mt8 is fully stored before + // this block reuses it, so TSS starts at mt0,row0. + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "j 9b\n" + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 2b\n" + // "8:\n" + + // Move from the top-right 32x32 block to the bottom-left block. + // A advances by one 32-row panel, B rewinds to its first column + // panel, and C advances diagonally from (0,32) to (32,0). + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // ".Lmatmul_block_%=:\n" + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + // Block 2 -> 3 moves from the top-right to the bottom-left panel. + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[blocks], %[blocks], -1\n" + // "beqz %[blocks], 8f\n" + + // // Advance horizontally until the N panel is exhausted. At the row + // // boundary, advance A by two tiles and rewind B to its first pair. + // "addi %[cols], %[cols], -1\n" + // "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + // "j 5f\n" + // "4:\n" + // "sub %[cols], %[b1p], %[b0p]\n" + // "add %[a0p], %[a0p], %[cols]\n" + // "add %[a1p], %[a1p], %[cols]\n" + // "slli %[loop], %[col_blocks], 1\n" + // "addi %[loop], %[loop], -1\n" + // "6:\n" + // "sub %[b0p], %[b0p], %[cols]\n" + // "sub %[b1p], %[b1p], %[cols]\n" + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 6b\n" + // "mv %[cols], %[col_blocks]\n" + // "slli %[loop], %[c_stride], 4\n" + // "addi %[loop], %[loop], -128\n" + // "sub %[loop], x0, %[loop]\n" + + // "5:\n" + // "sub %[c00], %[c00], %[loop]\n" + // "sub %[c01], %[c01], %[loop]\n" + // "sub %[c10], %[c10], %[loop]\n" + // "sub %[c11], %[c11], %[loop]\n" + // "j .Lmatmul_block_%=\n" + + // Flattened successor block. The previous mt8 is fully stored before + // this block reuses it, so TSS starts at mt0,row0. + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + // "j 9b\n" + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Final block: compact 16-row mt8 store loop. + // "7:\n" + // "li %[loop], 16\n" + // "2:\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 2b\n" + // "8:\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/kernel/matmul.h new file mode 100644 index 000000000..e635de107 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/kernel/matmul.h @@ -0,0 +1,31 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/main.c new file mode 100644 index 000000000..fda2c31d6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/main.c @@ -0,0 +1,310 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "matmul.h" +#include DATAHEADER +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + +enum { + M_CHUNK = CHUNK_SIZE, + N_CHUNK = CHUNK_SIZE, + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, + uint32_t panel_tiles, + uint32_t valid_tiles, + uint32_t valid_k, + uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) { + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + } + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { + const uint32_t first_tile = mb * A_TILES; + const uint32_t tile_count = + (matmul_l.M + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(A_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { + const uint32_t first_tile = nb * B_TILES; + const uint32_t tile_count = + (matmul_l.N + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(B_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); +} + +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + // This unrolled kernel executes exactly 256 original FP16 K elements. + _Static_assert(K_CHUNK == 256, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + const uint32_t job = iter / kb_count; + const uint32_t mb = job / nb_count; + const uint32_t nb = job % nb_count; + const uint32_t kb = iter % kb_count; + const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/script/gen_data.py new file mode 100644 index 000000000..697a422a6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_256.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_256.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/script/matmul_64_64_256.json new file mode 100644 index 000000000..db3d003b3 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K256/script/matmul_64_64_256.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/kernel/matmul.c new file mode 100644 index 000000000..c023d457f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/kernel/matmul.c @@ -0,0 +1,12236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 1, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Complete the block before changing any A/B/C panel pointer. The + // former cross-block path deferred this mt8 store while beginning the + // next block, but its FP32 register preload was incomplete. + + // // block transition + // "addi %[blocks], %[blocks], -1\n" + // "beqz %[blocks], 8f\n" + + // // Advance horizontally until the N panel is exhausted. At the row + // // boundary, advance A by two tiles and rewind B to its first pair. + // "addi %[cols], %[cols], -1\n" + // "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + // Flattened successor block. The previous mt8 is fully stored before + // this block reuses it, so TSS starts at mt0,row0. + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "j 9b\n" + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 2, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 2b\n" + // "8:\n" + + // Move from the top-right 32x32 block to the bottom-left block. + // A advances by one 32-row panel, B rewinds to its first column + // panel, and C advances diagonally from (0,32) to (32,0). + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // ".Lmatmul_block_%=:\n" + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + // Block 2 -> 3 moves from the top-right to the bottom-left panel. + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 3, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[blocks], %[blocks], -1\n" + // "beqz %[blocks], 8f\n" + + // // Advance horizontally until the N panel is exhausted. At the row + // // boundary, advance A by two tiles and rewind B to its first pair. + // "addi %[cols], %[cols], -1\n" + // "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + // "j 5f\n" + // "4:\n" + // "sub %[cols], %[b1p], %[b0p]\n" + // "add %[a0p], %[a0p], %[cols]\n" + // "add %[a1p], %[a1p], %[cols]\n" + // "slli %[loop], %[col_blocks], 1\n" + // "addi %[loop], %[loop], -1\n" + // "6:\n" + // "sub %[b0p], %[b0p], %[cols]\n" + // "sub %[b1p], %[b1p], %[cols]\n" + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 6b\n" + // "mv %[cols], %[col_blocks]\n" + // "slli %[loop], %[c_stride], 4\n" + // "addi %[loop], %[loop], -128\n" + // "sub %[loop], x0, %[loop]\n" + + // "5:\n" + // "sub %[c00], %[c00], %[loop]\n" + // "sub %[c01], %[c01], %[loop]\n" + // "sub %[c10], %[c10], %[loop]\n" + // "sub %[c11], %[c11], %[loop]\n" + // "j .Lmatmul_block_%=\n" + + // Flattened successor block. The previous mt8 is fully stored before + // this block reuses it, so TSS starts at mt0,row0. + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "vtfmm.tvv mt0, v0, v16\n" + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + // "j 9b\n" + // "9:\n" + // "mv %[loop], %[middle_k_groups]\n" + // "beqz %[loop], 3f\n" + // "1:\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 1b\n" + // "3:\n" + + // Block 4, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // Final block: compact 16-row mt8 store loop. + // "7:\n" + // "li %[loop], 16\n" + // "2:\n" + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // "addi %[loop], %[loop], -1\n" + // "bnez %[loop], 2b\n" + // "8:\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/kernel/matmul.h new file mode 100644 index 000000000..e635de107 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/kernel/matmul.h @@ -0,0 +1,31 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/main.c new file mode 100644 index 000000000..4ba920fd4 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/main.c @@ -0,0 +1,310 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "matmul.h" +#include DATAHEADER +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + +enum { + M_CHUNK = CHUNK_SIZE, + N_CHUNK = CHUNK_SIZE, + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, + uint32_t panel_tiles, + uint32_t valid_tiles, + uint32_t valid_k, + uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) { + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + } + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { + const uint32_t first_tile = mb * A_TILES; + const uint32_t tile_count = + (matmul_l.M + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(A_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { + const uint32_t first_tile = nb * B_TILES; + const uint32_t tile_count = + (matmul_l.N + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(B_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); +} + +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + // This unrolled kernel executes exactly 512 original FP16 K elements. + _Static_assert(K_CHUNK == 512, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + const uint32_t job = iter / kb_count; + const uint32_t mb = job / nb_count; + const uint32_t nb = job % nb_count; + const uint32_t kb = iter % kb_count; + const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/script/gen_data.py new file mode 100644 index 000000000..e2951a040 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_512.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_512.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/script/matmul_64_64_512.json new file mode 100644 index 000000000..4a78f5d6f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/store_interleave/K512/script/matmul_64_64_512.json @@ -0,0 +1,11 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/kernel/matmul.c similarity index 55% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/kernel/matmul.c rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/kernel/matmul.c index 09e445239..84df5ac60 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/kernel/matmul.c @@ -16,19 +16,20 @@ // // Author: Pei-Yu Lin, ETH Zürich -// fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) { const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); - uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); @@ -36,86 +37,134 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma asm volatile( // Block 1, K-Group 1 - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=1, twiden=1 - "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE - // we dont need preload + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" - + // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" @@ -126,68 +175,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 2 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -199,68 +216,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -272,68 +257,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -345,68 +298,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -418,68 +339,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -491,68 +380,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -564,68 +421,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -637,68 +462,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -710,68 +503,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -783,68 +544,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -856,68 +585,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -929,68 +626,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1002,131 +667,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 15+16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -1136,17 +753,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -1154,22 +767,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -1179,17 +788,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -1197,38 +802,50 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[loop], %[b1p], %[b0p]\n" @@ -1239,73 +856,117 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f // Block 2, K-Group 1 // vtfmm mt0 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" "vtfmm.tvv mt0, v0, v16\n" + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v6, v22\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v7, v23\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt4 - "vtzero mt4\n" "vtfmm.tvv mt4, v0, v24\n" + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v30\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" // vtfmm mt12 - "vtzero mt12\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v9, v25\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // vtfmm mt8 - "vtzero mt8\n" + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt8, v8, v16\n" "vtfmm.tvv mt8, v9, v17\n" "vtfmm.tvv mt8, v10, v18\n" @@ -1314,14 +975,10 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "addi %[loop], %[loop], -128\n" "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" @@ -1334,46 +991,26 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1385,68 +1022,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1458,68 +1063,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1531,68 +1104,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1604,68 +1145,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1677,68 +1186,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1750,68 +1227,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1823,68 +1268,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1896,68 +1309,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1969,68 +1350,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2042,68 +1391,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2115,68 +1432,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2188,131 +1473,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 15+16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -2322,17 +1559,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -2340,22 +1573,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -2365,17 +1594,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -2383,38 +1608,50 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[loop], %[b1p], %[b0p]\n" @@ -2428,48 +1665,77 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[b1p], %[b1p], %[loop]\n" // Block 3, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" @@ -2480,37 +1746,48 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // Block 3, K-Group 2 // vtfmm mt0 @@ -2518,55 +1795,31 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2578,68 +1831,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2651,68 +1872,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2724,68 +1913,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2797,68 +1954,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2870,68 +1995,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2943,68 +2036,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3016,68 +2077,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3089,68 +2118,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3162,68 +2159,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3235,68 +2200,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3308,68 +2241,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3381,131 +2282,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 15+16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -3515,17 +2368,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -3533,22 +2382,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -3558,17 +2403,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -3576,38 +2417,50 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[loop], %[b1p], %[b0p]\n" @@ -3617,48 +2470,77 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "add %[b1p], %[b1p], %[loop]\n" // Block 4, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" @@ -3669,36 +2551,47 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // Block 4, K-Group 2 // vtfmm mt0 @@ -3706,55 +2599,31 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3766,68 +2635,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3839,68 +2676,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3912,68 +2717,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3985,68 +2758,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4058,68 +2799,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4131,68 +2840,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4204,68 +2881,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4277,68 +2922,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4350,68 +2963,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4423,68 +3004,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4496,68 +3045,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4569,129 +3086,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" // Block 4, K-Group 15+16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -4701,17 +3172,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -4719,22 +3186,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -4744,17 +3207,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -4762,22 +3221,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -4787,13 +3242,9 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "lui %[tss], 0x40000\n" @@ -4815,20 +3266,23 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - // [blocks] "+r"(block_counter), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), // [cols] "+&r"(col_counter), [loop] "=&r"(loop_counter), [vl] "=&r"(vl_counter) : + [bias_base] "r"((uintptr_t)C), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [mtype] "r"(matrix_mtype), diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/kernel/matmul.h new file mode 100644 index 000000000..e635de107 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/kernel/matmul.h @@ -0,0 +1,31 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/main.c new file mode 100644 index 000000000..da2ca42bb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/main.c @@ -0,0 +1,310 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "matmul.h" +#include DATAHEADER +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + +enum { + M_CHUNK = CHUNK_SIZE, + N_CHUNK = CHUNK_SIZE, + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, + uint32_t panel_tiles, + uint32_t valid_tiles, + uint32_t valid_k, + uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) { + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + } + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { + const uint32_t first_tile = mb * A_TILES; + const uint32_t tile_count = + (matmul_l.M + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(A_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { + const uint32_t first_tile = nb * B_TILES; + const uint32_t tile_count = + (matmul_l.N + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(B_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); +} + +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + // This unrolled kernel executes exactly 128 original FP16 K elements. + _Static_assert(K_CHUNK == 128, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + const uint32_t job = iter / kb_count; + const uint32_t mb = job / nb_count; + const uint32_t nb = job % nb_count; + const uint32_t kb = iter % kb_count; + const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/script/gen_data.py new file mode 100644 index 000000000..92841c402 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/script/matmul_64_64_128.json new file mode 100644 index 000000000..3a9efc194 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K128/script/matmul_64_64_128.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/kernel/matmul.c similarity index 53% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/kernel/matmul.c rename to sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/kernel/matmul.c index b96a7ab34..84b415d08 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/kernel/matmul.c @@ -16,19 +16,20 @@ // // Author: Pei-Yu Lin, ETH Zürich -// fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) { const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); - uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); @@ -36,86 +37,134 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma asm volatile( // Block 1, K-Group 1 - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=1, twiden=1 - "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE - // we dont need preload + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" @@ -126,68 +175,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 2 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -199,68 +216,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -272,68 +257,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -345,68 +298,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -418,68 +339,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -491,68 +380,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -564,68 +421,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -637,68 +462,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -710,68 +503,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -783,68 +544,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -856,68 +585,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -929,68 +626,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1002,68 +667,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1075,68 +708,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 15 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1148,68 +749,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1221,68 +790,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 17 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1294,68 +831,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 18 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1367,68 +872,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 19 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1440,68 +913,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 20 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1513,68 +954,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 21 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1586,68 +995,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 22 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1659,68 +1036,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 23 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1732,68 +1077,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 24 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1805,68 +1118,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 25 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1878,68 +1159,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 26 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -1951,68 +1200,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 27 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2024,68 +1241,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 28 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2097,68 +1282,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 29 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2170,131 +1323,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 30 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 1, K-Group 31+32 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -2304,17 +1409,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -2322,22 +1423,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -2347,17 +1444,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -2365,38 +1458,50 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[loop], %[b1p], %[b0p]\n" @@ -2407,73 +1512,117 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f // Block 2, K-Group 1 // vtfmm mt0 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" "vtfmm.tvv mt0, v0, v16\n" + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v6, v22\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v7, v23\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt4 - "vtzero mt4\n" "vtfmm.tvv mt4, v0, v24\n" + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v30\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" // vtfmm mt12 - "vtzero mt12\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v9, v25\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // vtfmm mt8 - "vtzero mt8\n" + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt8, v8, v16\n" "vtfmm.tvv mt8, v9, v17\n" "vtfmm.tvv mt8, v10, v18\n" @@ -2482,14 +1631,10 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "addi %[loop], %[loop], -128\n" "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" @@ -2502,46 +1647,26 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2553,68 +1678,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2626,68 +1719,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2699,68 +1760,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2772,68 +1801,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2845,68 +1842,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2918,68 +1883,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -2991,68 +1924,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3064,68 +1965,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3137,68 +2006,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3210,68 +2047,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3283,68 +2088,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3356,68 +2129,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3429,68 +2170,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 15 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3502,68 +2211,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3575,68 +2252,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 17 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3648,68 +2293,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 18 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3721,68 +2334,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 19 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3794,68 +2375,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 20 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3867,68 +2416,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 21 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -3940,68 +2457,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 22 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4013,68 +2498,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 23 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4086,68 +2539,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 24 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4159,68 +2580,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 25 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4232,68 +2621,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 26 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4305,68 +2662,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 27 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4378,68 +2703,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 28 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4451,68 +2744,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 29 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4524,131 +2785,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 30 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 2, K-Group 31+32 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -4658,17 +2871,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -4676,22 +2885,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -4701,17 +2906,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -4719,38 +2920,50 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[loop], %[b1p], %[b0p]\n" @@ -4764,48 +2977,77 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[b1p], %[b1p], %[loop]\n" // Block 3, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" @@ -4816,37 +3058,48 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // Block 3, K-Group 2 // vtfmm mt0 @@ -4854,55 +3107,31 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4914,68 +3143,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -4987,68 +3184,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5060,68 +3225,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5133,68 +3266,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5206,68 +3307,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5279,68 +3348,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5352,68 +3389,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5425,68 +3430,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5498,68 +3471,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5571,68 +3512,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5644,68 +3553,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5717,68 +3594,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5790,68 +3635,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 15 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5863,68 +3676,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -5936,68 +3717,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 17 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6009,68 +3758,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 18 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6082,68 +3799,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 19 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6155,68 +3840,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 20 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6228,68 +3881,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 21 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6301,68 +3922,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 22 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6374,68 +3963,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 23 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6447,68 +4004,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 24 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6520,68 +4045,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 25 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6593,68 +4086,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 26 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6666,68 +4127,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 27 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6739,68 +4168,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 28 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6812,68 +4209,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 29 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -6885,131 +4250,83 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 30 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 3, K-Group 31+32 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -7019,17 +4336,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -7037,22 +4350,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -7062,17 +4371,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -7080,38 +4385,50 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[loop], %[b1p], %[b0p]\n" @@ -7121,48 +4438,77 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "add %[b1p], %[b1p], %[loop]\n" // Block 4, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" @@ -7173,36 +4519,47 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // Block 4, K-Group 2 // vtfmm mt0 @@ -7210,55 +4567,31 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7270,68 +4603,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 3 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7343,68 +4644,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 4 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7416,68 +4685,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 5 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7489,68 +4726,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 6 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7562,68 +4767,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 7 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7635,68 +4808,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 8 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7708,68 +4849,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 9 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7781,68 +4890,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 10 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7854,68 +4931,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 11 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -7927,68 +4972,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 12 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8000,68 +5013,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 13 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8073,68 +5054,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 14 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8146,68 +5095,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 15 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8219,68 +5136,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 16 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8292,68 +5177,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 17 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8365,68 +5218,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 18 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8438,68 +5259,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 19 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8511,68 +5300,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 20 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8584,68 +5341,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 21 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8657,68 +5382,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 22 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8730,68 +5423,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 23 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8803,68 +5464,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 24 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8876,68 +5505,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 25 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -8949,68 +5546,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 26 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -9022,68 +5587,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 27 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -9095,68 +5628,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 28 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -9168,68 +5669,36 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 29 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt12 @@ -9241,129 +5710,81 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt12, v11, v27\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 30 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v9, v25\n" "vtfmm.tvv mt12, v10, v26\n" "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" // Block 4, K-Group 31+32 // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v8, v24\n" "vtfmm.tvv mt0, v9, v25\n" "vtfmm.tvv mt0, v10, v26\n" "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v2, v18\n" "vtfmm.tvv mt4, v3, v19\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -9373,17 +5794,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v11, v27\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt12 @@ -9391,22 +5808,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v2, v18\n" "vtfmm.tvv mt12, v3, v19\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -9416,17 +5829,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v11, v27\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 @@ -9434,22 +5843,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v2, v18\n" "vtfmm.tvv mt8, v3, v19\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -9459,13 +5864,9 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v11, v27\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "lui %[tss], 0x40000\n" @@ -9487,20 +5888,23 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - // [blocks] "+r"(block_counter), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), // [cols] "+&r"(col_counter), [loop] "=&r"(loop_counter), [vl] "=&r"(vl_counter) : + [bias_base] "r"((uintptr_t)C), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [mtype] "r"(matrix_mtype), diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/kernel/matmul.h new file mode 100644 index 000000000..e635de107 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/kernel/matmul.h @@ -0,0 +1,31 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/main.c new file mode 100644 index 000000000..fda2c31d6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/main.c @@ -0,0 +1,310 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "matmul.h" +#include DATAHEADER +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + +enum { + M_CHUNK = CHUNK_SIZE, + N_CHUNK = CHUNK_SIZE, + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, + uint32_t panel_tiles, + uint32_t valid_tiles, + uint32_t valid_k, + uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) { + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + } + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { + const uint32_t first_tile = mb * A_TILES; + const uint32_t tile_count = + (matmul_l.M + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(A_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { + const uint32_t first_tile = nb * B_TILES; + const uint32_t tile_count = + (matmul_l.N + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(B_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); +} + +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + // This unrolled kernel executes exactly 256 original FP16 K elements. + _Static_assert(K_CHUNK == 256, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + const uint32_t job = iter / kb_count; + const uint32_t mb = job / nb_count; + const uint32_t nb = job % nb_count; + const uint32_t kb = iter % kb_count; + const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/script/gen_data.py new file mode 100644 index 000000000..f72dea613 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_256.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_256.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/script/matmul_64_64_256.json new file mode 100644 index 000000000..db3d003b3 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K256/script/matmul_64_64_256.json @@ -0,0 +1,14 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Parameters for a MatMul +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/data/layer.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/kernel/matmul.c new file mode 100644 index 000000000..042c57f25 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/kernel/matmul.c @@ -0,0 +1,10844 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +{ + const uint32_t tile_stride = TE * K; + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t tss; + uintptr_t bias_tss, bias_ptr; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=2 + "msettn x0, %[vl]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + // Block 1: preload mt0 bias from C[0][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 0\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 1: preload mt4 bias from C[0][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 64\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 1: preload mt8 bias from C[16][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4096\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + // Block 1: preload mt12 bias from C[16][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4160\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 2: preload mt0 bias from C[0][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 128\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtfmm.tvv mt0, v0, v16\n" + // Block 2: preload mt4 bias from C[0][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" + // Block 2: preload mt12 bias from C[16][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 4288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // vtfmm mt8 + // Block 2: preload mt8 bias from C[16][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 4224\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[vl]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 3: preload mt0 bias from C[32][0]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8192\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "lui %[tss], 0x40000\n" // mt8 + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 3: preload mt4 bias from C[32][16]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8256\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 3: preload mt12 bias from C[48][16]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12352\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + // Block 3: preload mt8 bias from C[48][0]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12288\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + // Block 4: preload mt0 bias from C[32][32]. + "li %[bias_tss], 0x00000000\n" + "li %[bias_ptr], 8320\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "lui %[tss], 0x40000\n" // mt8 + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + // Block 4: preload mt4 bias from C[32][48]. + "li %[bias_tss], 0x20000000\n" + "li %[bias_ptr], 8384\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + // Block 4: preload mt12 bias from C[48][48]. + "li %[bias_tss], 0x60000000\n" + "li %[bias_ptr], 12480\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + // Block 4: preload mt8 bias from C[48][32]. + "li %[bias_tss], 0x40000000\n" + "li %[bias_ptr], 12416\n" + "add %[bias_ptr], %[bias_base], %[bias_ptr]\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + "vtle32 %[bias_tss], (%[bias_ptr])\n" "add %[bias_ptr], %[bias_ptr], %[c_stride]\n" "addi %[bias_tss], %[bias_tss], 1\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt4, v11, v27\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt12, v11, v27\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtfmm.tvv mt8, v11, v27\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+&r"(a0p), + [a1p] "+&r"(a1p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [bias_tss] "=&r"(bias_tss), + [bias_ptr] "=&r"(bias_ptr), + [tss] "=&r"(tss), + [c00] "+&r"(c00), + [c01] "+&r"(c01), + [c10] "+&r"(c10), + [c11] "+&r"(c11), + // [blocks] "+&r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [bias_base] "r"((uintptr_t)C), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [col_blocks] "r"((uintptr_t)col_blocks) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/kernel/matmul.h new file mode 100644 index 000000000..e635de107 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/kernel/matmul.h @@ -0,0 +1,31 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/main.c b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/main.c new file mode 100644 index 000000000..4ba920fd4 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/main.c @@ -0,0 +1,310 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "matmul.h" +#include DATAHEADER +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} + +enum { + M_CHUNK = CHUNK_SIZE, + N_CHUNK = CHUNK_SIZE, + // K_CHUNK and all valid_k values count the original FP16 K dimension. + K_CHUNK = MATMUL_K_CHUNK, + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static const float zero_c_row[N_CHUNK] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, + uint32_t panel_tiles, + uint32_t valid_tiles, + uint32_t valid_k, + uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) { + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + } + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { + const uint32_t first_tile = mb * A_TILES; + const uint32_t tile_count = + (matmul_l.M + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(A_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { + const uint32_t first_tile = nb * B_TILES; + const uint32_t tile_count = + (matmul_l.N + TE - 1) / TE; + const uint32_t valid_tiles = first_tile < tile_count + ? min_u32(B_TILES, + tile_count - first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *src = valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + + (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, + matmul_l.K); +} + +// Only the first reduction chunk receives Caddend; later partials start at zero. +static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, + uint32_t kb) { + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_c_row, N_CHUNK * sizeof(float), + N_CHUNK * sizeof(float), 0, M_CHUNK); + const uint32_t rows = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t cols = chunk_extent(matmul_l.N, nb, N_CHUNK); + if (kb != 0 || rows == 0 || cols == 0) + return clear_tid; + snrt_dma_wait(clear_tid); + const float *src = Caddend_dram + + (size_t)mb * M_CHUNK * matmul_l.N + nb * N_CHUNK; + return snrt_dma_start_2d(dst, src, cols * sizeof(float), + N_CHUNK * sizeof(float), + matmul_l.N * sizeof(float), rows); +} + +static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, + uint32_t mb, uint32_t nb, uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + fill_a((uint32_t *)(base + A_OFFSET), mb, kb); + fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + return fill_c((float *)(base + C_OFFSET), mb, nb, kb); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + // This unrolled kernel executes exactly 512 original FP16 K elements. + _Static_assert(K_CHUNK == 512, "K_CHUNK must match the unrolled kernel"); + if (matmul_l.K == 0 || matmul_l.K % K_CHUNK != 0) { + printf("GEMM requires complete unrolled K chunks\n"); + return 2; + } + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 GEMM allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + const uint32_t job = iter / kb_count; + const uint32_t mb = job / nb_count; + const uint32_t nb = job % nb_count; + const uint32_t kb = iter % kb_count; + const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); + const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + // Preserve the computed C panel until its output DMA completes. + snrt_dma_wait(output_tid); + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 GEMM %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/script/gen_data.py new file mode 100644 index 000000000..37b53b32e --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/script/gen_data.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_512.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_512.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + Caddend = torch.randn((M, N), dtype=torch.float32).numpy() + A = A.numpy() + B = B.numpy() + # Match packed-pair SDOTP: two FP16 products and one FP32 addend. + C = Caddend.copy() + for pair_k in range(0, K, 2): + pair = (A[:, pair_k, None].astype(np.float64) * + B[None, pair_k, :].astype(np.float64) + + A[:, pair_k + 1, None].astype(np.float64) * + B[None, pair_k + 1, :].astype(np.float64)) + C = (C.astype(np.float64) + pair).astype(np.float32) + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "const matmul_layer matmul_l = {", + f" .M = {M},", + f" .N = {N},", + f" .K = {K},", + "};", + "", + "// FP32 GEMM golden reference: C = A*B + Caddend", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "// FP32 row-major initial addend for every output element.", + f'static float Caddend_dram[{M}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_f32(Caddend) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/script/matmul_64_64_512.json new file mode 100644 index 000000000..4a78f5d6f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-gemm-unit/unrolled/vsetvl/K512/script/matmul_64_64_512.json @@ -0,0 +1,11 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// SPDX-License-Identifier: Apache-2.0 +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/CMakeLists.txt b/sw/vme/apps/tew32/fp16-matmul-vse/CMakeLists.txt new file mode 100644 index 000000000..978e44e9e --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/CMakeLists.txt @@ -0,0 +1,143 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +set(_fp16_vse_te16_shapes + "64x64x128" + "64x64x256" + "64x64x512") + +set(_fp16_vse_te32_shapes + "128x128x128" + "128x128x256") + +function(add_fp16_vse_variant precision variant source_dir) + if(NOT EXISTS "${source_dir}/main.c" OR + NOT EXISTS "${source_dir}/kernel/matmul.c" OR + NOT EXISTS "${source_dir}/kernel/matmul.h" OR + NOT EXISTS "${source_dir}/script/gen_data.py" OR + NOT EXISTS "${source_dir}/data/layer.h") + message(STATUS "Skipping FP16 VSE ${precision}/${variant}: source tree is incomplete") + return() + endif() + + if(ARGC GREATER 3) + set(_variant_shapes ${ARGN}) + elseif(precision STREQUAL "te32") + set(_variant_shapes ${_fp16_vse_te32_shapes}) + else() + set(_variant_shapes ${_fp16_vse_te16_shapes}) + endif() + + if(precision STREQUAL "te32") + set(_tile_edge 32) + else() + set(_tile_edge 16) + endif() + + foreach(_shape IN LISTS _variant_shapes) + string(REPLACE "x" ";" _dims "${_shape}") + list(GET _dims 0 _m) + list(GET _dims 1 _n) + list(GET _dims 2 _k) + + # Generate only variants whose two input panels fit in TCDM. + math(EXPR _panel_bytes + "16 * ${_tile_edge} * ${_k} + 64 * ${_tile_edge} * ${_tile_edge}") + math(EXPR _double_buffer_bytes "2 * ${_panel_bytes}") + if(_double_buffer_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping FP16 VSE ${precision}/${variant} M${_m} N${_n} K${_k}: " + "double buffering requires ${_double_buffer_bytes} bytes, " + "but TCDM provides ${SNRT_TCDM_SIZE} bytes") + continue() + endif() + set(_buffer_count 2) + + set(_variant_build_dir + "${CMAKE_CURRENT_BINARY_DIR}/generated/${precision}/${variant}/K${_k}") + set(_variant_script_dir "${_variant_build_dir}/script") + set(_variant_data_dir "${source_dir}/data") + set(_variant_script "${source_dir}/script/gen_data.py") + + file(MAKE_DIRECTORY "${_variant_script_dir}") + + set(_config "${_variant_script_dir}/matmul_${_m}_${_n}_${_k}.json") + file(WRITE "${_config}" + "{ M: ${_m}, N: ${_n}, K: ${_k}, TM: ${_tile_edge}, TN: ${_tile_edge}, seed: 42 }\n") + + set(_header "${source_dir}/data/data_${_m}_${_n}_${_k}.h") + set(_data_header "data/data_${_m}_${_n}_${_k}.h") + set(_target + "fp16-matmul-vse-${precision}-${variant}_M${_m}_N${_n}_K${_k}") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${_variant_data_dir}" + COMMAND ${Python3_EXECUTABLE} "${_variant_script}" -c "${_config}" + DEPENDS "${_variant_script}" "${_config}" + "${source_dir}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${source_dir}/main.c") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${source_dir}" + "${source_dir}/kernel" + "${source_dir}/data") + target_compile_definitions(${_test_target} PRIVATE + MATMUL_BUFFER_COUNT=${_buffer_count} + DATAHEADER="${_data_header}") + endif() + endforeach() +endfunction() + +foreach(_precision IN ITEMS ${VME_TILE_VARIANT}) + if(_precision STREQUAL "te16") + # te16 is the active validation target. Keep all rolled scheduling + # variants aligned with the passing VTSE target matrix. + add_fp16_vse_variant(${_precision} rolled-store-afterK + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/store-afterK" + ${_fp16_vse_te16_shapes}) + add_fp16_vse_variant(${_precision} rolled-store-interleave + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/store-interleave" + ${_fp16_vse_te16_shapes}) + add_fp16_vse_variant(${_precision} rolled-vsetvl + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/vsetvl" + ${_fp16_vse_te16_shapes}) + add_fp16_vse_variant(${_precision} rolled-store-group + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/store-group" + ${_fp16_vse_te16_shapes}) + + # The te16 unrolled kernels are specialized by K. Keep their + # existing source hierarchy and expose the same K128/K256/K512 ELF + # matrix as the corresponding VTSE unrolled targets. + foreach(_k IN ITEMS 128 256 512) + set(_unrolled_shape "64x64x${_k}") + add_fp16_vse_variant(${_precision} unrolled-store-afterK + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/unrolled/store_afterK/K${_k}" + "${_unrolled_shape}") + add_fp16_vse_variant(${_precision} unrolled-store-interleave + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/unrolled/store_interleave/K${_k}" + "${_unrolled_shape}") + add_fp16_vse_variant(${_precision} unrolled-vsetvl + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/unrolled/vsetvl/K${_k}" + "${_unrolled_shape}") + endforeach() + else() + # Build all te32 LMUL=2 scheduling variants. + add_fp16_vse_variant(${_precision} rolled-store-afterK + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/store-afterK" + ${_fp16_vse_te32_shapes}) + add_fp16_vse_variant(${_precision} rolled-store-interleave + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/store-interleave" + ${_fp16_vse_te32_shapes}) + add_fp16_vse_variant(${_precision} rolled-vsetvl + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/vsetvl" + ${_fp16_vse_te32_shapes}) + endif() +endforeach() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c new file mode 100644 index 000000000..56b9c4999 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c @@ -0,0 +1,362 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t row_counter; + uintptr_t col_counter; + uintptr_t loop_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + const uintptr_t tile_bytes = operand_bytes * (K_CHUNK / 2); + const uintptr_t store_rows_bytes = TE * c_stride; + const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; + const uintptr_t row_c_advance = + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + // The first unrolled K-group is intentionally omitted from the middle + // loop schedule used by this store-afterK variant. + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + "mv %[rows], %[ROW_BLOCKS]\n" + ".Lrow_%=:\n" + "mv %[cols], %[COL_BLOCKS]\n" + ".Lcol_%=:\n" + // K-Group 1 + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "li %[loop], 16\n" + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "vtfmm.tvv mt4, v10, v26\n" + "vtfmm.tvv mt4, v11, v27\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v1, v17\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v25\n" + "vtfmm.tvv mt8, v10, v26\n" + "vtfmm.tvv mt8, v11, v27\n" + + // Store the completed 32x32 block only after all K-groups finish. + // The tile order is mt0, mt4, mt12, mt8. + "mv %[tss], x0\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "li %[loop], 8\n" + ".Lstore_mt0_%=:\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" + "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" + "add %[c00], %[c00], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt0_%=\n" + + "lui %[tss], 0x20000\n" + "li %[loop], 8\n" + ".Lstore_mt4_%=:\n" + "vtmv.v.t v8, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" + "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" + "add %[c01], %[c01], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt4_%=\n" + + "lui %[tss], 0x40000\n" + "li %[loop], 8\n" + ".Lstore_mt12_%=:\n" + "vtmv.v.t v16, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vtmv.v.t v17, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v16, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v17, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt12_%=\n" + + "lui %[tss], 0x60000\n" + "li %[loop], 8\n" + ".Lstore_mt8_%=:\n" + "vtmv.v.t v24, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vtmv.v.t v25, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v24, (%[c11])\n" + "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v25, (%[c11])\n" + "add %[c11], %[c11], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt8_%=\n" + + // Advance to the next 32-column block, or to the next 32-row block. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], .Lnext_row_%=\n" + + // Horizontal step: A is reused, Bpack advances by one tile, and C moves + // from the completed 32-column block to the next one. + "sub %[a0p], %[a0p], %[tile_bytes]\n" + "sub %[a1p], %[a1p], %[tile_bytes]\n" + "add %[b0p], %[b0p], %[tile_bytes]\n" + "add %[b1p], %[b1p], %[tile_bytes]\n" + "sub %[c00], %[c00], %[col_c_back]\n" + "sub %[c01], %[c01], %[col_c_back]\n" + "sub %[c10], %[c10], %[col_c_back]\n" + "sub %[c11], %[c11], %[col_c_back]\n" + "j .Lcol_%=\n" + + ".Lnext_row_%=:\n" + "addi %[rows], %[rows], -1\n" + "beqz %[rows], .Ldone_%=\n" + + // Vertical step: A advances to the next row pair, Bpack returns to the + // first column block, and C advances to the next row pair. + "add %[a0p], %[a0p], %[tile_bytes]\n" + "add %[a1p], %[a1p], %[tile_bytes]\n" + "sub %[b0p], %[b0p], %[row_b_rewind]\n" + "sub %[b1p], %[b1p], %[row_b_rewind]\n" + "add %[c00], %[c00], %[row_c_advance]\n" + "add %[c01], %[c01], %[row_c_advance]\n" + "add %[c10], %[c10], %[row_c_advance]\n" + "add %[c11], %[c11], %[row_c_advance]\n" + "j .Lrow_%=\n" + + ".Ldone_%=:\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [rows] "=&r"(row_counter), + [cols] "=&r"(col_counter), + [loop] "=&r"(loop_counter) + : + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_bytes] "r"(tile_bytes), + [col_c_back] "r"(col_c_back), + [row_c_advance] "r"(row_c_advance), + [row_b_rewind] "r"(row_b_rewind), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [middle_k_groups] "r"(middle_k_groups) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/gen_data.py new file mode 100644 index 000000000..97f79602f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-afterK/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/kernel/matmul.c new file mode 100644 index 000000000..01babd046 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/kernel/matmul.c @@ -0,0 +1,190 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32( + const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + const uintptr_t c_stride = 128 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t vtype = 0xC8; // e16, m1, tk=2 widening SDOTP + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + const uintptr_t bias_vl = 8 * TE; // m8: two contiguous 64-element FP32 rows + const uintptr_t operand_vector_bytes = TE * sizeof(uint32_t); + const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + + // Process each 16-row output block independently; no state crosses blocks. + for (uint32_t nb = 0; nb < 4; nb += 1) { + const uint32_t mb0 = nb * TE; + { + // Initialize and compute one complete 16-row output block. + uintptr_t a0p = (uintptr_t)(Atpack + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(Bpack + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(Bpack + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtzero mt0" : : : "memory"); + asm volatile("vtzero mt4" : : : "memory"); + asm volatile("vtzero mt8" : : : "memory"); + asm volatile("vtzero mt12" : : : "memory"); + + // Use the already-advanced A pointer as the loop induction variable. + const uintptr_t a_middle_end = a0p + (K_CHUNK / 8 - 1) * operand_group_bytes; + while (a0p != a_middle_end) { + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b1p) : "v24", "v25", "v26", "v27", "memory"); b1p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("vle32.v v16, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v16", "memory"); + asm volatile("vle32.v v17, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v17", "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vle32.v v0, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v0", "memory"); + asm volatile("vle32.v v1, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v1", "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + // Keep the pointer comparison; otherwise Clang adds a separate countdown. + asm volatile("" : "+r"(a0p)); + } + + // Final K group: move each completed tile row to the vector registers. + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vle32.v v2, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v2", "memory"); + asm volatile("vle32.v v3, (%0)\naddi %0, %0, %1" : "+r"(a0p) : "i"(operand_vector_bytes) : "v3", "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vle32.v v18, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v18", "memory"); + asm volatile("vle32.v v19, (%0)\naddi %0, %0, %1" : "+r"(b0p) : "i"(operand_vector_bytes) : "v19", "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b1p) : "v24", "v25", "v26", "v27", "memory"); b1p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v24" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v16, (%0)" : : "r"(b2p) : "v16", "v17", "v18", "v19", "memory"); b2p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v27" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v16" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b3p) : "v24", "v25", "v26", "v27", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v17" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v3, v19" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v25" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt12, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v3, v27" : : : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vse32.v v8, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("vse32.v v16, (%0)\n" "addi %0, %0, %1" : "+r"(ptr) : "i"(c_stride) : "memory"); + + // Rows 4-15: four contiguous 64-element rows per batch. + // Keep the store epilogue compact for the instruction cache. + for (uint32_t row = 4; row < TE; row += 4) { + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v8, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v8", "memory"); + asm volatile("vtmv.v.t v12, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v12", "memory"); + asm volatile("vtmv.v.t v9, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v9", "memory"); + asm volatile("vtmv.v.t v13, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v13", "memory"); + asm volatile("vtmv.v.t v10, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v10", "memory"); + asm volatile("vtmv.v.t v14, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v14", "memory"); + asm volatile("vtmv.v.t v11, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v11", "memory"); + asm volatile("vtmv.v.t v15, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v15", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vse32.v v8, (%0)\n" "addi %0, %0, %1\n" + : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(store_mtype), "r"(store_vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v16, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v16", "memory"); + asm volatile("vtmv.v.t v20, %0\naddi %0, %0, 1" : "+r"(tss0) : : "v20", "memory"); + asm volatile("vtmv.v.t v17, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v17", "memory"); + asm volatile("vtmv.v.t v21, %0\naddi %0, %0, 1" : "+r"(tss4) : : "v21", "memory"); + asm volatile("vtmv.v.t v18, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v18", "memory"); + asm volatile("vtmv.v.t v22, %0\naddi %0, %0, 1" : "+r"(tss8) : : "v22", "memory"); + asm volatile("vtmv.v.t v19, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v19", "memory"); + asm volatile("vtmv.v.t v23, %0\naddi %0, %0, 1" : "+r"(tss12) : : "v23", "memory"); + asm volatile("vsetvli x0, %0, e32, m8, ta, ma" : : "r"(bias_vl) : "memory"); + asm volatile("vse32.v v16, (%0)\n" "addi %0, %0, %1\n" + : "+r"(ptr) : "i"(c_stride) : "memory"); + } + } + } +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/gen_data.py new file mode 100644 index 000000000..49212bc76 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 8 or K % 8: + parser.error("packed store-group kernel requires real K >= 8 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// One e32,m4 load transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-group/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c new file mode 100644 index 000000000..e5f5bdae9 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c @@ -0,0 +1,563 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + const uintptr_t rewind_b0_bytes = operand_bytes * (TE / 2); + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + ".Lmatmul_block_%=:\n" + // K-Group 1 + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "li %[loop], 16\n" + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + // Defer mt8 to the next block's compute/store overlap. The final + // block branches to the compact drain loop below. + "lui %[tss], 0x40000\n" + "addi %[blocks], %[blocks], -1\n" + "beqz %[blocks], 7f\n" + + // Advance horizontally until the N panel is exhausted. Atpack the row + // boundary, advance A by two tiles and rewind Bpack to its first pair. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "j 5f\n" + "4:\n" + "sub %[cols], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[cols]\n" + "add %[a1p], %[a1p], %[cols]\n" + "slli %[loop], %[COL_BLOCKS], 1\n" + "addi %[loop], %[loop], -1\n" + "6:\n" + "sub %[b0p], %[b0p], %[cols]\n" + "sub %[b1p], %[b1p], %[cols]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 6b\n" + "mv %[cols], %[COL_BLOCKS]\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + + "5:\n" + + // Cross-Block Compute-Store overlapped (no control-flow path) + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "j 9b\n" + + // Final block: compact 16-row mt8 store loop. + "7:\n" + "li %[loop], 8\n" + "2:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vtmv.v.t v25, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v24, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v25, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 2b\n" + "8:\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [blocks] "+r"(block_counter), + [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [rewind_b0_bytes] "r"(rewind_b0_bytes), + [middle_k_groups] "r"(middle_k_groups), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/gen_data.py new file mode 100644 index 000000000..a6be95e41 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/store-interleave/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c new file mode 100644 index 000000000..fcabdea4d --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c @@ -0,0 +1,537 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + const uintptr_t rewind_b0_bytes = operand_bytes * (TE / 2); + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + ".Lmatmul_block_%=:\n" + // K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt0 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // A0 v0..v7 and B0 v16..v23 are already loaded; do not advance again. + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + // Defer mt8 to the next block's compute/store overlap. The final + // block branches to the compact drain loop below. + "lui %[tss], 0x40000\n" + "addi %[blocks], %[blocks], -1\n" + "beqz %[blocks], 7f\n" + + // Advance horizontally until the N panel is exhausted. Atpack the row + // boundary, advance A by two tiles and rewind Bpack to its first pair. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "j 5f\n" + "4:\n" + "sub %[cols], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[cols]\n" + "add %[a1p], %[a1p], %[cols]\n" + "slli %[loop], %[COL_BLOCKS], 1\n" + "addi %[loop], %[loop], -1\n" + "6:\n" + "sub %[b0p], %[b0p], %[cols]\n" + "sub %[b1p], %[b1p], %[cols]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 6b\n" + "mv %[cols], %[COL_BLOCKS]\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + + "5:\n" + + // Cross-Block Compute-Store overlapped (no control-flow path) + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "j 9b\n" + + // Final block: compact 16-row mt8 store loop. + "7:\n" + "li %[loop], 8\n" + "2:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vtmv.v.t v25, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v24, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v25, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 2b\n" + "8:\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [blocks] "+r"(block_counter), + [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [rewind_b0_bytes] "r"(rewind_b0_bytes), + [middle_k_groups] "r"(middle_k_groups), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/gen_data.py new file mode 100644 index 000000000..31dc2f2e0 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/rolled/vsetvl/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c new file mode 100644 index 000000000..721483f1f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c @@ -0,0 +1,3329 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py new file mode 100644 index 000000000..97f79602f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/matmul_64_64_64.json similarity index 57% rename from sw/vme/apps/tew32/fp32-matmul-vle-vse/script/gemm_64_64_64.json rename to sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/matmul_64_64_64.json index 9780457eb..dc71b114d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/script/gemm_64_64_64.json +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K128/script/matmul_64_64_64.json @@ -2,15 +2,13 @@ // Licensed under the Apache License, Version 2.0, see LICENSE for details. // SPDX-License-Identifier: Apache-2.0 // -// Author: Pei-Yu Lin, ETH Zürich -// Parameters for a GEMM +// Parameters for a MatMul { - kernel: "VME_GEMM", + kernel: "MatMul", M: 64, N: 64, K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) + TM: 16, + TN: 16, seed: 42 -} \ No newline at end of file +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c new file mode 100644 index 000000000..13c111831 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c @@ -0,0 +1,5953 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py new file mode 100644 index 000000000..25c9cc9b8 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_256.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_256.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K256/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/kernel/matmul.c new file mode 100644 index 000000000..f8e841f82 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/kernel/matmul.c @@ -0,0 +1,11201 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 63 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/gen_data.py new file mode 100644 index 000000000..c0d8fab5a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_512.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_512.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/matmul_64_64_256.json new file mode 100644 index 000000000..b1b3633fc --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K512/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c new file mode 100644 index 000000000..2249feb16 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c @@ -0,0 +1,2017 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 1 -> 2: return to row 0 and advance from columns 0-31 to 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 2 -> 3: advance to rows 32-63 and return to columns 0-31. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // Block 3 -> 4: remain on rows 32-63 and advance to columns 32-63. + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + "mv %[tss], x0\n" // mt0 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + "lui %[tss], 0x20000\n" // mt4 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v1, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v3, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v5, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v7, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v9, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v11, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v13, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v15, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + + "lui %[tss], 0x60000\n" // mt12 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v1, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v3, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v5, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v7, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v9, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v11, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v13, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v15, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" // mt8 + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py new file mode 100644 index 000000000..02230b5ee --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_64.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_64.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_afterK/K64/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c new file mode 100644 index 000000000..5434bc7c1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c @@ -0,0 +1,3514 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + // Final block: compact 16-row mt8 store loop. + "lui %[tss], 0x40000\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py new file mode 100644 index 000000000..a6be95e41 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/matmul_64_64_64.json similarity index 53% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/script/gemm_64_64_64.json rename to sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/matmul_64_64_64.json index 3a26d4cde..dc71b114d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/script/gemm_64_64_64.json +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K128/script/matmul_64_64_64.json @@ -2,16 +2,13 @@ // Licensed under the Apache License, Version 2.0, see LICENSE for details. // SPDX-License-Identifier: Apache-2.0 // -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. +// Parameters for a MatMul { - kernel: "VME_GEMM", + kernel: "MatMul", M: 64, N: 64, K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) + TM: 16, + TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c new file mode 100644 index 000000000..5af9fdfce --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c @@ -0,0 +1,6138 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + // Final block: compact 16-row mt8 store loop. + "lui %[tss], 0x40000\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py new file mode 100644 index 000000000..3fd8e2f67 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_256.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_256.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K256/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/kernel/matmul.c new file mode 100644 index 000000000..0873360b4 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/kernel/matmul.c @@ -0,0 +1,11386 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + // Final block: compact 16-row mt8 store loop. + "lui %[tss], 0x40000\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/gen_data.py new file mode 100644 index 000000000..8f5a9ca49 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_512.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_512.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/matmul_64_64_256.json new file mode 100644 index 000000000..b1b3633fc --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K512/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c new file mode 100644 index 000000000..41ca46baf --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c @@ -0,0 +1,2202 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "add %[c00], %[c00], %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" + "add %[c01], %[c01], %[loop]\n" + "add %[c10], %[c10], %[loop]\n" + "add %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + // vtfmm mt0 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "lui %[tss], 0x40000\n" // mt8 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v11, v27\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + // Final block: compact 16-row mt8 store loop. + "lui %[tss], 0x40000\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py new file mode 100644 index 000000000..dc3446ff6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_64.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_64.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/store_interleave/K64/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c new file mode 100644 index 000000000..e28b84f2f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c @@ -0,0 +1,3161 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // Block 4, K-Group 15+16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + // [blocks] "+r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py new file mode 100644 index 000000000..31dc2f2e0 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/matmul_64_64_128.json new file mode 100644 index 000000000..522a92ceb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/matmul_64_64_128.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 128, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/matmul_64_64_64.json similarity index 53% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/script/gemm_64_64_64.json rename to sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/matmul_64_64_64.json index 3a26d4cde..dc71b114d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/script/gemm_64_64_64.json +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K128/script/matmul_64_64_64.json @@ -2,16 +2,13 @@ // Licensed under the Apache License, Version 2.0, see LICENSE for details. // SPDX-License-Identifier: Apache-2.0 // -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. +// Parameters for a MatMul { - kernel: "VME_GEMM", + kernel: "MatMul", M: 64, N: 64, K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) + TM: 16, + TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c new file mode 100644 index 000000000..0119e791e --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c @@ -0,0 +1,5783 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 31+32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + // [blocks] "+r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py new file mode 100644 index 000000000..5ba1ab97f --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_256.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_256.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/script/matmul_64_64_256.json new file mode 100644 index 000000000..4edd605d1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K256/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/kernel/matmul.c new file mode 100644 index 000000000..3ad2a583b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/kernel/matmul.c @@ -0,0 +1,11031 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 7 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 9 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 10 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 11 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 12 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 13 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 14 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 15 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 16 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 17 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 18 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 19 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 20 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 21 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 22 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 23 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 24 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 25 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 26 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 27 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 28 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 29 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 30 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 31 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 32 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 33 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 34 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 35 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 36 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 37 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 38 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 39 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 40 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 41 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 42 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 43 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 44 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 45 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 46 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 47 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 48 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 49 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 50 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 51 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 52 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 53 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 54 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 55 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 56 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 57 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 58 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 59 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 60 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 61 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 62 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 4, K-Group 63+64 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "addi %[b0p], %[b0p], -512\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + "lui %[tss], 0x40000\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v4, %[tss]\n" "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + // [blocks] "+r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/gen_data.py new file mode 100644 index 000000000..4356164c2 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_512.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_512.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/matmul_64_64_256.json new file mode 100644 index 000000000..b1b3633fc --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/matmul_64_64_256.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 256, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/matmul_64_64_512.json new file mode 100644 index 000000000..2cec13cd6 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K512/script/matmul_64_64_512.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 512, TM: 16, TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c new file mode 100644 index 000000000..758cd0050 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c @@ -0,0 +1,2136 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t loop_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD0; + + asm volatile( + + // Block 1, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 2 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 1, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 1, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 2, K-Group 1 + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vtfmm.tvv mt8, v9, v17\n" + "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v11, v19\n" + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + + // Block 2, K-Group 2 + // vtfmm mt0 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 2, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 2, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[loop]\n" + "add %[a1p], %[a1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + "sub %[b0p], %[b0p], %[loop]\n" + "sub %[b1p], %[b1p], %[loop]\n" + + // Block 3, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + // vtfmm mt0 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "sub %[loop], x0, %[loop]\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 3, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + + // Block 3, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + + // Block 4, K-Group 1 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + // vtfmm mt0 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v25\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v26\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v27\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + // vtfmm mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "slli %[loop], %[c_stride], 4\n" + "addi %[loop], %[loop], -128\n" + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 2 + // vtfmm mt0 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 3 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 4 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 5 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v9, v25\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // Block 4, K-Group 6 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v9, v25\n" + "vtfmm.tvv mt12, v10, v26\n" + "vtfmm.tvv mt12, v11, v27\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // Block 4, K-Group 7+8 + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v9, v25\n" + "vtfmm.tvv mt0, v10, v26\n" + "vtfmm.tvv mt0, v11, v27\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v3, v19\n" + "vse32.v v1, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v3, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v5, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v7, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v11, v27\n" + "vse32.v v9, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v11, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v13, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vse32.v v14, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vse32.v v15, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v2, v18\n" + "vse32.v v16, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v3, v19\n" + "vse32.v v17, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vse32.v v22, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vse32.v v24, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v11, v27\n" + "vse32.v v25, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vse32.v v30, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vse32.v v31, (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v1, v17\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v16, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vse32.v v16, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v17, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v3, v19\n" + "vse32.v v17, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v18, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v18, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v19, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v19, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v20, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v20, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v21, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v21, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v22, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vse32.v v22, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v23, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v23, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v9, v25\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v24, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v10, v26\n" + "vse32.v v24, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v25, %[tss]\n" "addi %[tss], %[tss], 1\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v11, v27\n" + "vse32.v v25, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v26, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v26, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v27, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v27, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v28, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v28, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v29, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v29, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vtmv.v.t v30, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v31, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v30, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + "vse32.v v31, (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" + + "lui %[tss], 0x40000\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v1, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v1, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v3, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v3, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v5, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v5, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v7, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v7, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v9, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v9, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v11, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v11, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v13, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v13, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vtmv.v.t v14, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vtmv.v.t v15, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v14, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + "vse32.v v15, (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" + + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + // [blocks] "+r"(block_counter), + // [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype) + // [rewind_b0_bytes] "r"(rewind_b0_bytes), + // [middle_k_groups] "r"(middle_k_groups), + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h new file mode 100644 index 000000000..22470e49b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 elements, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py new file mode 100644 index 000000000..eda7ec7b5 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_64_64_64.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 16): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_64_64_64.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][16]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m4 and VL=64 transfers four packed K-pair registers.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][16], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te16/unrolled/vsetvl/K64/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c new file mode 100644 index 000000000..5bd0bbb48 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c @@ -0,0 +1,279 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=32, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t row_counter; + uintptr_t col_counter; + uintptr_t loop_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC9; // e16, m2, ta, ma + const uintptr_t tile_bytes = operand_bytes * (K_CHUNK / 2); + const uintptr_t store_rows_bytes = TE * c_stride; + const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; + const uintptr_t row_c_advance = + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD1; + + asm volatile( + "mv %[rows], %[ROW_BLOCKS]\n" + ".Lrow_%=:\n" + "mv %[cols], %[COL_BLOCKS]\n" + ".Lcol_%=:\n" + // K-Group 1 + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "li %[loop], 32\n" + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v0, v16\n" + "vtfmm.tvv mt4, v2, v18\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v8, v24\n" + "vtfmm.tvv mt4, v10, v26\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt12 + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v2, v18\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v10, v26\n" + "addi %[b0p], %[b0p], -512\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt8 + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v2, v18\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtfmm.tvv mt8, v10, v26\n" + + // Store each completed 32x32 accumulator after all K-groups finish. + // LMUL=2 vector instructions use even register-group bases only. + "mv %[tss], x0\n" // mt0 -> c00 + "li %[loop], 32\n" + ".Lstore_mt0_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" + "add %[c00], %[c00], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt0_%=\n" + + "lui %[tss], 0x20000\n" // mt4 -> c01 + "li %[loop], 32\n" + ".Lstore_mt4_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" + "add %[c01], %[c01], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt4_%=\n" + + "lui %[tss], 0x40000\n" // mt8 -> c10 + "li %[loop], 32\n" + ".Lstore_mt8_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt8_%=\n" + + "lui %[tss], 0x60000\n" // mt12 -> c11 + "li %[loop], 32\n" + ".Lstore_mt12_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" + "add %[c11], %[c11], %[c_stride]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], .Lstore_mt12_%=\n" + + // Advance to the next 32-column block, or to the next 32-row block. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], .Lnext_row_%=\n" + + // Horizontal step: A is reused, Bpack advances by one tile, and C moves + // from the completed 32-column block to the next one. + "sub %[a0p], %[a0p], %[tile_bytes]\n" + "sub %[a1p], %[a1p], %[tile_bytes]\n" + "add %[b0p], %[b0p], %[tile_bytes]\n" + "add %[b1p], %[b1p], %[tile_bytes]\n" + "sub %[c00], %[c00], %[col_c_back]\n" + "sub %[c01], %[c01], %[col_c_back]\n" + "sub %[c10], %[c10], %[col_c_back]\n" + "sub %[c11], %[c11], %[col_c_back]\n" + "j .Lcol_%=\n" + + ".Lnext_row_%=:\n" + "addi %[rows], %[rows], -1\n" + "beqz %[rows], .Ldone_%=\n" + + // Vertical step: A advances to the next row pair, Bpack returns to the + // first column block, and C advances to the next row pair. + "add %[a0p], %[a0p], %[tile_bytes]\n" + "add %[a1p], %[a1p], %[tile_bytes]\n" + "sub %[b0p], %[b0p], %[row_b_rewind]\n" + "sub %[b1p], %[b1p], %[row_b_rewind]\n" + "add %[c00], %[c00], %[row_c_advance]\n" + "add %[c01], %[c01], %[row_c_advance]\n" + "add %[c10], %[c10], %[row_c_advance]\n" + "add %[c11], %[c11], %[row_c_advance]\n" + "j .Lrow_%=\n" + + ".Ldone_%=:\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [rows] "=&r"(row_counter), + [cols] "=&r"(col_counter), + [loop] "=&r"(loop_counter) + : + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_bytes] "r"(tile_bytes), + [col_c_back] "r"(col_c_back), + [row_c_advance] "r"(row_c_advance), + [row_b_rewind] "r"(row_b_rewind), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [middle_k_groups] "r"(middle_k_groups) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h new file mode 100644 index 000000000..de3f24ebf --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 32, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/gen_data.py new file mode 100644 index 000000000..ad62dc0bb --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_128_128_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 32): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_128_128_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][32]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with VL=16 transfers one 512-bit packed K pair.", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][32], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_128.json new file mode 100644 index 000000000..4d98bfb94 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_128.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 128, TM: 32, TN: 32, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_256.json new file mode 100644 index 000000000..0b86a203a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_256.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 256, TM: 32, TN: 32, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_64.json new file mode 100644 index 000000000..b5decff47 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_128_128_64.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-afterK/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c new file mode 100644 index 000000000..c88dacdd5 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c @@ -0,0 +1,386 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=32, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; + uintptr_t loop_counter; + uintptr_t store_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC9; // e16, m2, ta, ma + const uintptr_t rewind_b0_bytes = 4 * operand_bytes; + const uintptr_t tile_bytes = operand_bytes * (K_CHUNK / 2); + const uintptr_t store_rows_bytes = TE * c_stride; + const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; + const uintptr_t row_c_advance = + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD1; + + asm volatile( + ".Lmatmul_block_%=:\n" + // K-Group 1 + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "li %[loop], 32\n" + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt0\n" + + // vtfmm mt0 + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v10, v26\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v2, v18\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v0, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v2, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v2, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v4, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v4, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v8, v24\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v6, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v6, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v8, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v10, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v10, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + + // Finish mt0 rows 28-31 with the already released v12 group. Avoid + // reusing v14 immediately after it is consumed by the final mt4 MAC. + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vtmv.v.t v12, %[tss]\n" "addi %[tss], %[tss], 1\n" + "vse32.v v12, (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v8, v24\n" + "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtfmm.tvv mt8, v10, v26\n" + + // Delay the right-half stores until mt12/mt8 have consumed every + // Bpack-side register group. This avoids overwriting live LMUL=2 operands. + "lui %[tss], 0x20000\n" // mt4 -> c01 + "li %[store], 32\n" + ".Lstore_mt4_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" + "add %[c01], %[c01], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_mt4_%=\n" + + "lui %[tss], 0x60000\n" // mt12 -> c11 + "li %[store], 32\n" + ".Lstore_mt12_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" + "add %[c11], %[c11], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_mt12_%=\n" + + // Defer mt8 to the next block's compute/store overlap. The final + // block branches to the VSE drain loop below. + "lui %[tss], 0x40000\n" + "addi %[blocks], %[blocks], -1\n" + "beqz %[blocks], 7f\n" + + // Advance horizontally until the N panel is exhausted. Atpack the row + // boundary, advance A by two tiles and rewind Bpack to its first pair. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], 4f\n" + "sub %[a0p], %[a0p], %[tile_bytes]\n" + "sub %[a1p], %[a1p], %[tile_bytes]\n" + "add %[b0p], %[b0p], %[tile_bytes]\n" + "add %[b1p], %[b1p], %[tile_bytes]\n" + "mv %[loop], %[col_c_back]\n" + "j 5f\n" + "4:\n" + "mv %[cols], %[COL_BLOCKS]\n" + "add %[a0p], %[a0p], %[tile_bytes]\n" + "add %[a1p], %[a1p], %[tile_bytes]\n" + "sub %[b0p], %[b0p], %[row_b_rewind]\n" + "sub %[b1p], %[b1p], %[row_b_rewind]\n" + "sub %[loop], x0, %[row_c_advance]\n" + + "5:\n" + + // Cross-block compute/store overlap. Compute the first K group of + // the next block while draining the preceding block's mt8. + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v2, v18\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + + "li %[store], 32\n" + ".Lstore_prev_mt8_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_prev_mt8_%=\n" + + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v2, v26\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + + // All four C pointers have now advanced by exactly 32 rows. + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "j 9b\n" + + // Final block: drain the deferred mt8 through legal LMUL=2 VSE. + "7:\n" + "li %[store], 32\n" + ".Lstore_final_mt8_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_final_mt8_%=\n" + "8:\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [blocks] "+r"(block_counter), + [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [store] "=&r"(store_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes] "r"(operand_bytes), + [tile_bytes] "r"(tile_bytes), + [col_c_back] "r"(col_c_back), + [row_c_advance] "r"(row_c_advance), + [row_b_rewind] "r"(row_b_rewind), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [rewind_b0_bytes] "r"(rewind_b0_bytes), + [middle_k_groups] "r"(middle_k_groups), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h new file mode 100644 index 000000000..de3f24ebf --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 32, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/gen_data.py new file mode 100644 index 000000000..947e50c11 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_128_128_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 32): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_128_128_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][32]: word[31:16]=even K, word[15:0]=odd K.", + "// Four consecutive e32,m1 loads transfer four packed K-pair groups (eight registers).", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][32], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_128.json new file mode 100644 index 000000000..4d98bfb94 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_128.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 128, TM: 32, TN: 32, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_256.json new file mode 100644 index 000000000..0b86a203a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_256.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 256, TM: 32, TN: 32, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_64.json new file mode 100644 index 000000000..b5decff47 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_128_128_64.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_64_64_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_64_64_64.json new file mode 100644 index 000000000..818502866 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/store-interleave/script/matmul_64_64_64.json @@ -0,0 +1 @@ +{ M: 64, N: 64, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/data/layer.h b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c new file mode 100644 index 000000000..6e996a6a0 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c @@ -0,0 +1,356 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp16->fp32 optimal kernel for TE=32, CE=8, and four accumulator tiles. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) +{ + uintptr_t a0p = (uintptr_t)Atpack; + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); + uintptr_t b0p = (uintptr_t)Bpack; + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); + uintptr_t tss; + const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; + const uintptr_t operand_bytes_m2 = operand_bytes; + const uintptr_t operand_bytes_m4 = 2 * operand_bytes; + const uintptr_t c_block_bytes = 2 * operand_bytes; + uintptr_t c00 = (uintptr_t)(C); + uintptr_t c01 = c00 + operand_bytes; + uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); + uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; + uintptr_t loop_counter; + uintptr_t store_counter; + uintptr_t vl_counter; + const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); + const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; + const uintptr_t matrix_vtype = 0xC9; // e16, m2, ta, ma + const uintptr_t rewind_b0_bytes = 2 * operand_bytes_m4; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; + + // VTMV requires SEW=TEW and twiden=1 in the controller. + const uintptr_t store_mtype = (TE << 10) | (1u << 5) | 1u; + const uintptr_t store_vtype = 0xD1; + + asm volatile( + ".Lmatmul_block_%=:\n" + // K-Group 1 + // LMUL=2: VLE/VTFMM use even registers only. The vtmv/vse32 store + // destinations below intentionally preserve the te16 schedule. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes_m4]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" // sew=16, tm=TE, tk=2, twiden=2 + "msettn x0, %[tile_n]\n" // msetmtype resets tn, so set full tn=TE + + // vtfmm mt0 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m2]\n" + "vtzero mt4\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m2]\n" + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m2]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m2]\n" + "vtzero mt8\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt12\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + "9:\n" + "mv %[loop], %[middle_k_groups]\n" + "beqz %[loop], 3f\n" + "1:\n" + // A0 v0..v7 and B0 v16..v23 are already loaded; do not advance again. + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m2]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m2]\n" + + // vtfmm mt4 + "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 + "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m2]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m2]\n" + + // vtfmm mt8 + "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 + "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 1b\n" + "3:\n" + + // K-Group N-1 & K-Group N + // vtfmm mt0 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt0, v8, v24\n" + "vtfmm.tvv mt0, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt4 + // vtse mt0 + "mv %[tss], x0\n" // mt0 + "vtfmm.tvv mt4, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v2, v18\n" + "vtfmm.tvv mt4, v8, v24\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt4, v10, v26\n" + + // vtfmm mt12 + // vtse mt4 + "lui %[tss], 0x20000\n" // mt4 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v0, v16\n" + "vtfmm.tvv mt12, v2, v18\n" + "vtfmm.tvv mt12, v8, v24\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt12, v10, v26\n" + + // vtfmm mt8 + // vtse mt12 + "lui %[tss], 0x60000\n" // mt12 + "vtfmm.tvv mt8, v0, v16\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v8, v24\n" + "vtfmm.tvv mt8, v10, v26\n" + + // Store the three non-deferred tiles. LMUL=2 requires an even + // vector-register group, and each TE32 tile has 32 complete rows. + "mv %[tss], x0\n" // mt0 -> c00 + "li %[store], 32\n" + ".Lstore_mt0_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c00])\n" + "add %[c00], %[c00], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_mt0_%=\n" + + "lui %[tss], 0x20000\n" // mt4 -> c01 + "li %[store], 32\n" + ".Lstore_mt4_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c01])\n" + "add %[c01], %[c01], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_mt4_%=\n" + + "lui %[tss], 0x60000\n" // mt12 -> c11 + "li %[store], 32\n" + ".Lstore_mt12_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c11])\n" + "add %[c11], %[c11], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_mt12_%=\n" + + // Defer mt8 to the next block's compute/store overlap. The final + // block branches to the compact drain loop below. + "lui %[tss], 0x40000\n" + "addi %[blocks], %[blocks], -1\n" + "beqz %[blocks], 7f\n" + + // Advance horizontally until the N panel is exhausted. Atpack the row + // boundary, advance A by two tiles and rewind Bpack to its first pair. + "addi %[cols], %[cols], -1\n" + "beqz %[cols], 4f\n" + "sub %[loop], %[b1p], %[b0p]\n" + "sub %[a0p], %[a0p], %[loop]\n" + "sub %[a1p], %[a1p], %[loop]\n" + "add %[b0p], %[b0p], %[loop]\n" + "add %[b1p], %[b1p], %[loop]\n" + "slli %[loop], %[c_stride], 5\n" + "sub %[loop], %[loop], %[c_block_bytes]\n" + "j 5f\n" + "4:\n" + "sub %[cols], %[b1p], %[b0p]\n" + "add %[a0p], %[a0p], %[cols]\n" + "add %[a1p], %[a1p], %[cols]\n" + "slli %[loop], %[COL_BLOCKS], 1\n" + "addi %[loop], %[loop], -1\n" + "6:\n" + "sub %[b0p], %[b0p], %[cols]\n" + "sub %[b1p], %[b1p], %[cols]\n" + "addi %[loop], %[loop], -1\n" + "bnez %[loop], 6b\n" + "mv %[cols], %[COL_BLOCKS]\n" + "slli %[loop], %[c_stride], 5\n" + "sub %[loop], %[loop], %[c_block_bytes]\n" + "sub %[loop], x0, %[loop]\n" + + "5:\n" + + // Cross-Block Compute-Store overlapped (no control-flow path) + // vtfmm mt0 + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes_m4]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "lui %[tss], 0x40000\n" // mt8 + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v2, v18\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // Store the preceding block's deferred mt8 while computing the next + // block. v8 is free until the A1 group is loaded below. + "li %[store], 32\n" + ".Lstore_prev_mt8_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v8, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v8, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_prev_mt8_%=\n" + + // vtfmm mt4 + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + "vtzero mt4\n" + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v2, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt12 + "vtzero mt12\n" + "vtfmm.tvv mt12, v8, v24\n" + "vtfmm.tvv mt12, v10, v26\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_n]\n" + + // vtfmm mt8 + "vtzero mt8\n" + "vtfmm.tvv mt8, v8, v16\n" + // "slli %[loop], %[c_stride], 5\n" + // "sub %[loop], %[loop], %[c_block_bytes]\n" + "sub %[c00], %[c00], %[loop]\n" + "sub %[c01], %[c01], %[loop]\n" + "vtfmm.tvv mt8, v10, v18\n" + "sub %[c10], %[c10], %[loop]\n" + "sub %[c11], %[c11], %[loop]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes_m4]\n" + "msetmtype %[mtype], %[vtype]\n" + "msettn x0, %[tile_n]\n" + "j 9b\n" + + // Final block: compact 32-row mt8 store loop. + "7:\n" + "li %[store], 32\n" + ".Lstore_final_mt8_%=:\n" + "msetmtype %[store_mtype], %[store_vtype]\n" "msettn x0, %[tile_n]\n" + "vtmv.v.t v0, %[tss]\n" + "addi %[tss], %[tss], 1\n" + "vse32.v v0, (%[c10])\n" + "add %[c10], %[c10], %[c_stride]\n" + "addi %[store], %[store], -1\n" + "bnez %[store], .Lstore_final_mt8_%=\n" + "8:\n" + : + [a0p] "+r"(a0p), + [a1p] "+r"(a1p), + [b0p] "+r"(b0p), + [b1p] "+r"(b1p), + [tss] "=&r"(tss), + [c00] "+r"(c00), + [c01] "+r"(c01), + [c10] "+r"(c10), + [c11] "+r"(c11), + [blocks] "+r"(block_counter), + [cols] "+&r"(col_counter), + [loop] "=&r"(loop_counter), + [store] "=&r"(store_counter), + [vl] "=&r"(vl_counter) + : + [c_stride] "r"(c_stride), + [operand_bytes_m2] "r"(operand_bytes_m2), + [operand_bytes_m4] "r"(operand_bytes_m4), + [c_block_bytes] "r"(c_block_bytes), + [tile_n] "r"((uintptr_t)TE), + [store_mtype] "r"(store_mtype), + [store_vtype] "r"(store_vtype), + [mtype] "r"(matrix_mtype), + [vtype] "r"(matrix_vtype), + [rewind_b0_bytes] "r"(rewind_b0_bytes), + [middle_k_groups] "r"(middle_k_groups), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) + : "memory"); +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h new file mode 100644 index 000000000..de3f24ebf --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h @@ -0,0 +1,52 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 32, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +// A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/main.c new file mode 100644 index 000000000..13711a609 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/main.c @@ -0,0 +1,236 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include DATAHEADER +#include "matmul.h" +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; +} + +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); + +_Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); + +static const uint32_t zero_tile_dram[TILE_INPUTS] + __attribute__((section(".dram"), aligned(64))) = {0}; + +static inline uint32_t min_u32(uint32_t a, uint32_t b) { + return a < b ? a : b; +} + +static inline uint32_t chunk_extent(uint32_t total, uint32_t block, + uint32_t chunk) { + const uint32_t offset = block * chunk; + return offset < total ? min_u32(chunk, total - offset) : 0; +} + +static inline uint32_t float_bits(float value) { + union { + float f; + uint32_t u; + } bits = {.f = value}; + return bits.u; +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { + if (valid_tiles == panel_tiles && valid_k == K_CHUNK && + src_tile_k == K_CHUNK) + return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); + + const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( + dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); + if (valid_tiles == 0 || valid_k == 0) + return clear_tid; + + const size_t copy_bytes = (valid_k / 2) * TE * sizeof(uint32_t); + const size_t src_stride = (src_tile_k / 2) * TE * sizeof(uint32_t); + return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, + valid_tiles); +} + +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; + const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); + const uint32_t k0 = kb * K_CHUNK; + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; + const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; + const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; + const uint32_t jobs = mb_count * nb_count; + const uint32_t iterations = jobs * kb_count; + + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp16xfp16->fp32 matmul allocation failure\n"); + return 2; + } + + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; + uint32_t current_buffer = 0; + + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); + snrt_dma_wait(input_tid[0]); + if (L1_BUFFER_COUNT == 2 && iterations > 1) { + const uint32_t next_job = 1 / kb_count; + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); + } + + uint32_t start_cycle = 0; + uint32_t end_cycle = 0; + for (uint32_t iter = 0; iter < iterations; ++iter) { + uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; + + if (iter == 0) + start_cycle = get_cycle(); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); + + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); + + const uint32_t next_iter = iter + 1; + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { + const uint32_t next_buffer = current_buffer ^ 1; + snrt_dma_wait(input_tid[next_buffer]); + + const uint32_t refill_iter = iter + 2; + if (refill_iter < iterations) { + const uint32_t refill_job = refill_iter / kb_count; + input_tid[current_buffer] = fill_buffer( + l1_base, current_buffer, refill_job / nb_count, + refill_job % nb_count, refill_iter % kb_count); + } + current_buffer = next_buffer; + } + } + snrt_dma_wait_all(); + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t mb = row / M_CHUNK; + const uint32_t nb = col / N_CHUNK; + const uint32_t job = mb * nb_count + nb; + const uint32_t local_row = row % M_CHUNK; + const uint32_t local_col = col % N_CHUNK; + float sum = 0.0f; + for (uint32_t kb = 0; kb < kb_count; ++kb) { + const float *partial = partials + + (size_t)(job * kb_count + kb) * C_ELEMENTS; + sum += partial[(size_t)local_row * N_CHUNK + local_col]; + } + const size_t index = (size_t)row * matmul_l.N + col; + const float actual = sum; + const float expected = Cref_dram[index]; + const float delta = actual > expected ? actual - expected + : expected - actual; + const float tolerance = 0.05f + 0.02f * + (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols ", row); + else + printf(","); + printf("%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + const uint32_t cycles = end_cycle - start_cycle; + const uint64_t useful_fmas = + (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_fmas = (uint64_t)cycles * CE * CE * 2; + const uint32_t utilization_bp = + peak_fmas != 0 + ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) + : 0; + + printf("\n=== TEW32 FP16xFP16->FP32 matmul %ux%ux%u " + "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K, + M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); + printf("Cycles: %u\n", cycles); + printf("Utilization: %u.%02u%%\n", utilization_bp / 100, + utilization_bp % 100); + printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); + printf("Errors: %u\n", errors); + printf("Status: %s\n", errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/gen_data.py new file mode 100644 index 000000000..3f1aa8542 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/gen_data.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Author: Pei-Yu Lin +# +# +# Usage: +# python3 script/gen_data.py -c script/matmul_128_128_128.json +# +# Output: +# data/data_{M}_{N}_{K}.h +# +# The config JSON must provide: M, N, K. +# Optional field: seed (default 42). + +import argparse +import pathlib + +import hjson +import numpy as np +import torch + +def array_to_cstr_f32(arr: np.ndarray) -> str: + """Flat float32 array → C brace-initializer with 'f' suffix.""" + return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" + +def array_to_cstr_u32(arr: np.ndarray) -> str: + return "{" + ", ".join(f"0x{int(x):08x}u" for x in arr.flat) + "}" + +def pack_pairs(a: np.ndarray, b: np.ndarray, te: int = 32): + """Words are {even K halfword, odd K halfword}; no FP32 conversion.""" + m, k = a.shape + assert b.shape[0] == k and k % 2 == 0 + n = b.shape[1] + assert m % te == 0 and n % te == 0 + abits = a.astype(np.float16).view(np.uint16).astype(np.uint32) + bbits = b.astype(np.float16).view(np.uint16).astype(np.uint32) + aw = (abits[:, 0::2] << 16) | abits[:, 1::2] + bw = (bbits[0::2, :] << 16) | bbits[1::2, :] + return (aw.reshape(m // te, te, k // 2).transpose(0, 2, 1).copy(), + bw.reshape(k // 2, n // te, te).transpose(1, 0, 2).copy()) + +def main(): + parser = argparse.ArgumentParser( + description="Generate FP16xFP16->FP32 VME GEMM benchmark data header" + ) + parser.add_argument( + "-c", "--cfg", + type=pathlib.Path, + required=True, + help="JSON/HJSON config file (e.g. script/matmul_128_128_128.json)", + ) + args = parser.parse_args() + + with args.cfg.open() as f: + param = hjson.loads(f.read()) + + M = int(param["M"]) + N = int(param["N"]) + K = int(param["K"]) + seed = int(param.get("seed", 42)) + if K < 24 or K % 8: + parser.error("packed VSE kernel requires real K >= 24 and divisible by 8") + + torch.manual_seed(seed) + + A = torch.randn((M, K), dtype=torch.float32).to(torch.float16) + B = torch.randn((K, N), dtype=torch.float32).to(torch.float16) + C = (A.to(torch.float32) @ B.to(torch.float32)).numpy() + A = A.numpy() + B = B.numpy() + + A_pack, B_pack = pack_pairs(A, B) + + lines = [ + "// Copyright 2026 ETH Zurich and University of Bologna.", + "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", + "// SPDX-License-Identifier: Apache-2.0", + "//", + "// Author: Pei-Yu Lin ", + "//", + "// Auto-generated by script/gen_data.py — do not edit by hand.", + "", + '#include "layer.h"', + '#include ', + "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", + "};", + "", + "// C[M*N] row-major fp32 golden reference, C = A*B", + f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', + array_to_cstr_f32(C) + ";", + "", + "", + "// A_pack[m_tile][K/2][32]: word[31:16]=even K, word[15:0]=odd K.", + "// vle32 with e32,m8 and VL=128 transfers four packed K-pair groups (eight registers).", + f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(A_pack) + ";", + "", + "", + "// B_pack[n_tile][K/2][32], same halfword order as A.", + f'static uint32_t Bpack_dram[{K//2}*{N}] __attribute__((section(".data"), aligned(64))) =', + array_to_cstr_u32(B_pack) + ";", + "", + ] + + out_dir = pathlib.Path(__file__).parent.parent / "data" + out_dir.mkdir(parents=True, exist_ok=True) + out_file = out_dir / f"data_{M}_{N}_{K}.h" + + with out_file.open("w") as f: + f.write("\n".join(lines)) + + print(f"Written: {out_file}") + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_128.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_128.json new file mode 100644 index 000000000..4d98bfb94 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_128.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 128, TM: 32, TN: 32, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_256.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_256.json new file mode 100644 index 000000000..0b86a203a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_256.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 256, TM: 32, TN: 32, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_64.json b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_64.json new file mode 100644 index 000000000..b5decff47 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-matmul-vse/te32/rolled/vsetvl/script/matmul_128_128_64.json @@ -0,0 +1 @@ +{ M: 128, N: 128, K: 64, seed: 42 } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/CMakeLists.txt b/sw/vme/apps/tew32/fp16-matmul-vtse/CMakeLists.txt index 8165c010c..b9974a5dd 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/CMakeLists.txt +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/CMakeLists.txt @@ -40,17 +40,18 @@ function(add_fp16_vtse_variant precision variant source_dir) set(_tile_edge 16) endif() - # Use double buffering only when both input panels fit in the - # 512-KiB TCDM. Larger panels use a single buffer and are refilled - # synchronously by main.c. + # Generate only variants whose two input panels fit in TCDM. math(EXPR _panel_bytes - "32 * ${_tile_edge} * ${_k} + 64 * ${_tile_edge} * ${_tile_edge}") + "16 * ${_tile_edge} * ${_k} + 64 * ${_tile_edge} * ${_tile_edge}") math(EXPR _double_buffer_bytes "2 * ${_panel_bytes}") - if(_double_buffer_bytes GREATER 524288) - set(_buffer_count 1) - else() - set(_buffer_count 2) + if(_double_buffer_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping FP16 VTSE ${precision}/${variant} M${_m} N${_n} K${_k}: " + "double buffering requires ${_double_buffer_bytes} bytes, " + "but TCDM provides ${SNRT_TCDM_SIZE} bytes") + continue() endif() + set(_buffer_count 2) set(_variant_build_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${precision}/${variant}/K${_k}") @@ -83,21 +84,18 @@ function(add_fp16_vtse_variant precision variant source_dir) add_custom_target(generate-${_target} DEPENDS "${_header}") add_snitch_test(${_target} "${source_dir}/main.c") add_dependencies(${_test_target} generate-${_target}) - set_target_properties(${_test_target} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/vme/apps") target_include_directories(${_test_target} PRIVATE "${source_dir}" "${source_dir}/kernel" "${source_dir}/data") target_compile_definitions(${_test_target} PRIVATE - MATMUL_K_CHUNK=${_k_chunk} MATMUL_BUFFER_COUNT=${_buffer_count} DATAHEADER="${_data_header}") endif() endforeach() endfunction() -foreach(_precision IN ITEMS te16 te32) +foreach(_precision IN ITEMS ${VME_TILE_VARIANT}) if(_precision STREQUAL "te32") set(_precision_shapes ${_fp16_vtse_te32_shapes}) set(_unrolled_ks 128 256) diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c index 7cc083dc5..8c98e67d2 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c @@ -19,15 +19,12 @@ #include "matmul.h" // fp16->fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -40,22 +37,22 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma - const uintptr_t tile_bytes = operand_bytes * (K / 2); + const uintptr_t tile_bytes = operand_bytes * (K_CHUNK / 2); const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; // Each packed A/B word contains two FP16 K rows. asm volatile( "msetmtype %[mtype], %[vtype]\n" "li %[loop], 16\n" "msettn x0, %[loop]\n" - "mv %[rows], %[row_blocks]\n" + "mv %[rows], %[ROW_BLOCKS]\n" ".Lrow_%=:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" ".Lcol_%=:\n" // K-Group 1 @@ -318,8 +315,8 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [cols] "=&r"(col_counter), [loop] "=&r"(loop_counter) : - [row_blocks] "r"((uintptr_t)row_blocks), - [col_blocks] "r"((uintptr_t)col_blocks), + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [tile_bytes] "r"(tile_bytes), diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h index 826b7de7c..18a1cfdd6 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts pairs, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py index f86159816..1d8dec1ed 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c index c5ecef8ab..fa7dbefe2 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c @@ -22,30 +22,27 @@ // Each 32-bit word contains two FP16 K rows for one SDOTP element. // K counts original FP16 rows. Retain cross-block compute/store overlap. -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t c_tile_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + c_tile_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; const uintptr_t rewind_b0_bytes = 8 * operand_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( "msetmtype %[mtype], %[vtype]\n" @@ -155,16 +152,16 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // accumulate both packed FP16 products "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // accumulate both packed FP16 products "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products + "vtfmm.tvv mt0, v1, v17\n" // accumulate both packed FP16 products "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // accumulate both packed FP16 products + "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" // accumulate both packed FP16 products "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v8, v24\n" + "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v9, v25\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" @@ -301,14 +298,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" @@ -377,14 +374,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtfmm.tvv mt8, v8, v16\n" "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v9, v17\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v10, v18\n" + "vtfmm.tvv mt8, v9, v17\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v11, v19\n" "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "j 9b\n" @@ -418,6 +415,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [rewind_b0_bytes] "r"(rewind_b0_bytes), [operand_bytes] "r"(operand_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h index 826b7de7c..18a1cfdd6 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts pairs, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py index b6f40834e..4831bf3c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c index 0fc609140..3a24d2bd5 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c @@ -23,23 +23,20 @@ // VL=64 and fills four consecutive vector registers; msetmtype then restores // the unchanged e16/tk=2/twiden=2 matrix configuration. -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t c_tile_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + c_tile_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); @@ -47,7 +44,7 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); const uintptr_t rewind_b0_bytes = 2 * operand_group_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -260,14 +257,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" @@ -367,6 +364,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [rewind_b0_bytes] "r"(rewind_b0_bytes), [operand_group_bytes] "r"(operand_group_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h index 826b7de7c..18a1cfdd6 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts pairs, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py index bd32b35a2..9d1226bcc 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c index ab4fe8d96..3dabd27cb 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -3044,6 +3043,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py index f86159816..1d8dec1ed 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c index be011d90d..71335208c 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -5667,6 +5666,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py index d86853e73..e1242201a 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.c index 57d72a1cd..4dd03fa33 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -10915,6 +10914,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/script/gen_data.py index 149ccbc69..00743a404 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_afterK/K512/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c index a2406b7b9..eb1b678b6 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -2430,14 +2429,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // "sub %[cols], %[b1p], %[b0p]\n" // "add %[a0p], %[a0p], %[cols]\n" // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" + // "slli %[loop], %[COL_BLOCKS], 1\n" // "addi %[loop], %[loop], -1\n" // "6:\n" // "sub %[b0p], %[b0p], %[cols]\n" // "sub %[b1p], %[b1p], %[cols]\n" // "addi %[loop], %[loop], -1\n" // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" + // "mv %[cols], %[COL_BLOCKS]\n" // "slli %[loop], %[c_stride], 4\n" // "addi %[loop], %[loop], -128\n" // "sub %[loop], x0, %[loop]\n" @@ -3283,6 +3282,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py index b6f40834e..4831bf3c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c index 0b652bee3..30a84d445 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -4590,14 +4589,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // "sub %[cols], %[b1p], %[b0p]\n" // "add %[a0p], %[a0p], %[cols]\n" // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" + // "slli %[loop], %[COL_BLOCKS], 1\n" // "addi %[loop], %[loop], -1\n" // "6:\n" // "sub %[b0p], %[b0p], %[cols]\n" // "sub %[b1p], %[b1p], %[cols]\n" // "addi %[loop], %[loop], -1\n" // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" + // "mv %[cols], %[COL_BLOCKS]\n" // "slli %[loop], %[c_stride], 4\n" // "addi %[loop], %[loop], -128\n" // "sub %[loop], x0, %[loop]\n" @@ -6163,6 +6162,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py index ec1cf169e..a37068c3c 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.c index b8be40586..3fd66f696 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -8910,14 +8909,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // "sub %[cols], %[b1p], %[b0p]\n" // "add %[a0p], %[a0p], %[cols]\n" // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" + // "slli %[loop], %[COL_BLOCKS], 1\n" // "addi %[loop], %[loop], -1\n" // "6:\n" // "sub %[b0p], %[b0p], %[cols]\n" // "sub %[b1p], %[b1p], %[cols]\n" // "addi %[loop], %[loop], -1\n" // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" + // "mv %[cols], %[COL_BLOCKS]\n" // "slli %[loop], %[c_stride], 4\n" // "addi %[loop], %[loop], -128\n" // "sub %[loop], x0, %[loop]\n" @@ -11923,6 +11922,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/script/gen_data.py index c1f55173f..3695d38a2 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/store_interleave/K512/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c index 24cb1aac3..1b9f9fb53 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -2981,6 +2980,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py index bd32b35a2..9d1226bcc 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c index d6fb42601..6dc3a8b57 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -5603,6 +5602,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py index 25ce3e657..980fbd9dd 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.c index ce655955f..7fa8b4f4c 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -10531,6 +10530,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.h index e635de107..e65a701c9 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/script/gen_data.py index 8b60f1750..925158742 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te16/unrolled/vsetvl/K512/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c index a7e206170..dfe9f52c7 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c @@ -19,15 +19,12 @@ #include "matmul.h" // fp16->fp32 optimal kernel for TE=32, CE=8, and four accumulator tiles. -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -40,77 +37,60 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma - const uintptr_t tile_bytes = operand_bytes * (K / 2); + const uintptr_t tile_bytes = operand_bytes * (K_CHUNK / 2); + const uintptr_t rewind_b0_bytes = 4 * operand_bytes; const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + // Each group consumes two packed pairs (four original FP16 K rows). + // The first and final two groups are unrolled. + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; // Each packed A/B word contains two FP16 K rows. asm volatile( "msetmtype %[mtype], %[vtype]\n" "li %[loop], 32\n" "msettn x0, %[loop]\n" - "mv %[rows], %[row_blocks]\n" + "mv %[rows], %[ROW_BLOCKS]\n" ".Lrow_%=:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" ".Lcol_%=:\n" // K-Group 1 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtzero mt0\n" // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // accumulate both packed FP16 products - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // accumulate both packed FP16 products "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // accumulate both packed FP16 products - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v0, v16\n" // accumulate both packed FP16 products + "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // accumulate both packed FP16 products - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // accumulate both packed FP16 products "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // accumulate both packed FP16 products + "vtfmm.tvv mt4, v2, v26\n" // accumulate both packed FP16 products "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // accumulate both packed FP16 products - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtzero mt8\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // accumulate both packed FP16 products - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // accumulate both packed FP16 products + "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // accumulate both packed FP16 products - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" "9:\n" "mv %[loop], %[middle_k_groups]\n" @@ -121,40 +101,24 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // accumulate both packed FP16 products - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // accumulate both packed FP16 products - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // accumulate both packed FP16 products "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // accumulate both packed FP16 products "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // accumulate both packed FP16 products - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // accumulate both packed FP16 products - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // accumulate both packed FP16 products "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // accumulate both packed FP16 products - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // accumulate both packed FP16 products - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 1b\n" "3:\n" @@ -163,75 +127,43 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // accumulate both packed FP16 products "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // accumulate both packed FP16 products + "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // accumulate both packed FP16 products "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" + "vtfmm.tvv mt0, v10, v26\n" "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" - "vle32.v v20, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v16\n" - "vle32.v v22, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v18\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v20\n" + "vtfmm.tvv mt4, v2, v18\n" "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v8, v24\n" - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v10, v26\n" "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v12, v28\n" + "vtfmm.tvv mt4, v10, v26\n" "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vle32.v v4, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt12 "vtfmm.tvv mt12, v0, v16\n" - "vle32.v v6, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v2, v18\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v4, v20\n" + "vtfmm.tvv mt12, v2, v18\n" + "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v6, v22\n" - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v14, v30\n" - "addi %[b0p], %[b0p], -1024\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // vtfmm mt8 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt8, v0, v16\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v2, v18\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v4, v20\n" + "vtfmm.tvv mt8, v2, v18\n" "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt8, v8, v24\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt8, v10, v26\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtfmm.tvv mt8, v14, v30\n" // Store the completed 64x64 block only after all K-groups finish. // The tile order is mt0, mt4, mt12, mt8. @@ -318,10 +250,11 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [cols] "=&r"(col_counter), [loop] "=&r"(loop_counter) : - [row_blocks] "r"((uintptr_t)row_blocks), - [col_blocks] "r"((uintptr_t)col_blocks), + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), + [rewind_b0_bytes] "r"(rewind_b0_bytes), [tile_bytes] "r"(tile_bytes), [col_c_back] "r"(col_c_back), [row_c_advance] "r"(row_c_advance), diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h index f0d8d6081..de3f24ebf 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py index 4c5b0bbd8..d71d72dde 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", @@ -98,7 +104,7 @@ def main(): "", "", "// A_pack[m_tile][K/2][32]: word[31:16]=even K, word[15:0]=odd K.", - "// vle32 with VL=16 transfers one 512-bit packed K pair.", + "// vle32 with VL=32 transfers one 1024-bit packed K pair across two registers.", f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', array_to_cstr_u32(A_pack) + ";", "", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c index 99e615a1f..50b371fea 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c @@ -22,30 +22,29 @@ // Each 32-bit word contains two FP16 K rows for one SDOTP element. // K counts original FP16 rows. Retain cross-block compute/store overlap. -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t c_tile_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + c_tile_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma const uintptr_t operand_bytes = 2 * sizeof(__fp16) * TE; - const uintptr_t rewind_b0_bytes = 8 * operand_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t rewind_b0_bytes = 4 * operand_bytes; + // Each group consumes two packed pairs (four original FP16 K rows). + // The first and final two groups are unrolled. + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( "msetmtype %[mtype], %[vtype]\n" @@ -55,54 +54,34 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtzero mt0\n" // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // accumulate both packed FP16 products - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // accumulate both packed FP16 products + "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // accumulate both packed FP16 products - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtzero mt4\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // accumulate both packed FP16 products - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // accumulate both packed FP16 products "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // accumulate both packed FP16 products + "vtfmm.tvv mt4, v2, v26\n" // accumulate both packed FP16 products "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // accumulate both packed FP16 products - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtzero mt8\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // accumulate both packed FP16 products - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // accumulate both packed FP16 products + "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // accumulate both packed FP16 products - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtzero mt12\n" // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" "9:\n" "mv %[loop], %[middle_k_groups]\n" @@ -113,40 +92,24 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // accumulate both packed FP16 products - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // accumulate both packed FP16 products - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" // vtfmm mt4 "vtfmm.tvv mt4, v0, v24\n" // accumulate both packed FP16 products "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v2, v26\n" // accumulate both packed FP16 products "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // accumulate both packed FP16 products - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // accumulate both packed FP16 products - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" // vtfmm mt8 "vtfmm.tvv mt8, v8, v16\n" // accumulate both packed FP16 products "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // accumulate both packed FP16 products - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // accumulate both packed FP16 products - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" // accumulate both packed FP16 products // vtfmm mt12 "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v10, v26\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 1b\n" "3:\n" @@ -155,41 +118,28 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" // accumulate both packed FP16 products "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // accumulate both packed FP16 products + "vtfmm.tvv mt0, v2, v18\n" // accumulate both packed FP16 products "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // accumulate both packed FP16 products "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" + "vtfmm.tvv mt0, v10, v26\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" - "vle32.v v20, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtfmm.tvv mt4, v0, v16\n" - "vle32.v v22, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v2, v18\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -201,18 +151,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v8, v24\n" - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v10, v26\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v14, v30\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -227,21 +173,19 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // vtfmm mt12 // vtse mt4 "lui %[tss], 0x20000\n" - "vle32.v v4, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtfmm.tvv mt12, v0, v16\n" - "vle32.v v6, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v2, v18\n" + "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" + "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -253,48 +197,38 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" + "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" // vtfmm mt8 // vtse mt12 "lui %[tss], 0x60000\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtfmm.tvv mt8, v0, v16\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v2, v18\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" + "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -306,16 +240,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt8, v8, v24\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" + "vtfmm.tvv mt8, v10, v26\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -349,14 +280,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 5\n" "addi %[loop], %[loop], -256\n" "sub %[loop], x0, %[loop]\n" @@ -364,34 +295,25 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "5:\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" // Cross-block compute/store overlap: drain the previous mt8 before reuse. // vtfmm mt0 "lui %[tss], 0x40000\n" // mt8 "vtzero mt0\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v0, v16\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v2, v18\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -402,23 +324,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtfmm.tvv mt4, v0, v24\n" - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" + "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" + "vtfmm.tvv mt4, v2, v26\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" // vtfmm mt12 "vtzero mt12\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -428,28 +345,22 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v14, v30\n" + "vtfmm.tvv mt12, v10, v26\n" // vtfmm mt8 "vtzero mt8\n" "vtfmm.tvv mt8, v8, v16\n" "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v10, v18\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" + "vtfmm.tvv mt8, v10, v18\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" "j 9b\n" // Final block: compact 32-row mt8 store loop. @@ -482,6 +393,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [rewind_b0_bytes] "r"(rewind_b0_bytes), [operand_bytes] "r"(operand_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h index f0d8d6081..de3f24ebf 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py index bdeb4a02c..bebb90d6e 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", @@ -98,7 +104,7 @@ def main(): "", "", "// A_pack[m_tile][K/2][32]: word[31:16]=even K, word[15:0]=odd K.", - "// Four consecutive e32,m1 loads transfer four packed K-pair groups (eight registers).", + "// Two vle32 loads with VL=32 transfer two packed K pairs (four registers).", f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', array_to_cstr_u32(A_pack) + ";", "", diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c index 8c690cdcf..8a0298811 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c @@ -19,40 +19,39 @@ #include "matmul.h" // fp16->fp32 optimal kernel for TE=32, CE=8, and four accumulator tiles. -// Each 32-bit element packs two FP16 reduction rows. An e32,m8 load has -// VL=128 and fills eight consecutive vector registers (four packed K pairs); msetmtype then restores +// Each 32-bit element packs two FP16 reduction rows. An e32,m4 load has +// VL=64 and fills four consecutive vector registers (two packed K pairs); msetmtype then restores // the unchanged e16/tk=2/twiden=2 matrix configuration. -__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)Atpack; - uintptr_t a1p = (uintptr_t)(Atpack + tile_stride); + uintptr_t a1p = (uintptr_t)(Atpack + TILE_STRIDE); uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); + uintptr_t b1p = (uintptr_t)(Bpack + TILE_STRIDE); uintptr_t tss; const uintptr_t c_tile_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + c_tile_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 2u; const uintptr_t matrix_vtype = 0xC8; // e16, m1, ta, ma - const uintptr_t operand_group_bytes = 4 * TE * sizeof(uint32_t); + const uintptr_t operand_group_bytes = 2 * TE * sizeof(uint32_t); const uintptr_t rewind_b0_bytes = 2 * operand_group_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + // Each group consumes two packed pairs (four original FP16 K rows). + // The first and final two groups are unrolled. + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( ".Lmatmul_block_%=:\n" - // K group 0: fill four MAC FIFO entries before the next load phase. - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + // K group 0: load two packed K pairs without touching implicit row-stride groups. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" @@ -60,25 +59,19 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtzero mt0\n" "vtfmm.tvv mt0, v0, v16\n" "vtfmm.tvv mt0, v2, v18\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v6, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtzero mt4\n" "vtfmm.tvv mt4, v0, v24\n" "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v6, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtzero mt8\n" "vtfmm.tvv mt8, v8, v16\n" "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v14, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" @@ -86,70 +79,55 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtzero mt12\n" "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v14, v30\n" "9:\n" "mv %[loop], %[middle_k_groups]\n" "beqz %[loop], 3f\n" "1:\n" - // Middle K groups: four consecutive VTFMMs per load/compute phase. - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + // Middle K groups: two consecutive VTFMMs per load/compute phase. + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v0, v16\n" "vtfmm.tvv mt0, v2, v18\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v6, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt4, v0, v24\n" "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v6, v30\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt8, v8, v16\n" "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v14, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt12, v8, v24\n" "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v14, v30\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 1b\n" "3:\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" // K-Group N-1 & K-Group N // vtfmm mt0 "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v2, v18\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v6, v22\n" + "vtfmm.tvv mt0, v2, v18\n" "vtfmm.tvv mt0, v8, v24\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" // vtfmm mt4 // vtse mt0 "mv %[tss], x0\n" // mt0 "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v2, v18\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -158,15 +136,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" + "vtfmm.tvv mt4, v2, v18\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -177,18 +154,16 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" + "vtfmm.tvv mt4, v10, v26\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -197,7 +172,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // vtse mt4 "lui %[tss], 0x20000\n" // mt4 "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v2, v18\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -206,14 +180,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" + "vtfmm.tvv mt12, v2, v18\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -223,21 +196,19 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "sub %[b0p], %[b0p], %[rewind_b0_bytes]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v12, v28\n" + "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -246,7 +217,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac // vtse mt12 "lui %[tss], 0x60000\n" // mt12 "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v2, v18\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -255,14 +225,13 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" + "vtfmm.tvv mt8, v2, v18\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -272,17 +241,15 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" + "vtfmm.tvv mt8, v10, v26\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -308,20 +275,20 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 5\n" "addi %[loop], %[loop], -256\n" "sub %[loop], x0, %[loop]\n" "5:\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" @@ -336,13 +303,11 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" + "vtfmm.tvv mt0, v2, v18\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" @@ -357,16 +322,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" + "vtfmm.tvv mt4, v2, v26\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" // vtfmm mt12 @@ -378,25 +341,21 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v14, v30\n" + "vtfmm.tvv mt12, v10, v26\n" // vtfmm mt8 "vtzero mt8\n" "vtfmm.tvv mt8, v8, v16\n" "sub %[c00], %[c00], %[loop]\n" "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v10, v18\n" "sub %[c10], %[c10], %[loop]\n" "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v14, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" + "vtfmm.tvv mt8, v10, v18\n" + "vsetvli %[vl], x0, e32, m4, ta, ma\n" "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_group_bytes]\n" "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" "j 9b\n" @@ -431,6 +390,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp16_fp32(const __fp16 *Atpac [rewind_b0_bytes] "r"(rewind_b0_bytes), [operand_group_bytes] "r"(operand_group_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h index f0d8d6081..de3f24ebf 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h @@ -25,7 +25,28 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, +}; + +enum { + TILE_INPUTS = TE * (K_CHUNK / 2), + TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, }; // A/B are packed FP16 halfwords; K counts original FP16 rows, C is FP32. -void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C, uint32_t M, uint32_t N, uint32_t K); +void matmul_fp16_fp32(const __fp16 *Atpack, const __fp16 *Bpack, float *C); diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/main.c index 1809bc596..13711a609 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/main.c +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/main.c @@ -19,46 +19,22 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - // K_CHUNK and all valid_k values count the original FP16 K dimension. - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * (K_CHUNK / 2), - TILE_BYTES = TILE_INPUTS * sizeof(uint32_t), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(MATMUL_K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); +_Static_assert(K_CHUNK % 8 == 0, "packed kernel requires K divisible by 8"); _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const uint32_t zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -81,15 +57,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + uint32_t *dst, const uint32_t *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -102,47 +75,37 @@ static snrt_dma_txid_t fill_panel(uint32_t *dst, const uint32_t *src, valid_tiles); } -static snrt_dma_txid_t fill_a(uint32_t *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_b(uint32_t *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const uint32_t *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * (matmul_l.K / 2) + k0 / 2) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, - matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((uint32_t *)(base + A_OFFSET), mb, kb); - return fill_b((uint32_t *)(base + B_OFFSET), nb, kb); + const uint32_t *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + const uint32_t *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K / 2) + k0 / 2) * TE; + + fill_panel((uint32_t *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((uint32_t *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -177,18 +140,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp16_fp32((const __fp16 *)(base + A_OFFSET), (const __fp16 *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py index a9739c3b7..d791fc141 100644 --- a/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py +++ b/sw/vme/apps/tew32/fp16-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py @@ -86,10 +86,16 @@ def main(): '#include "layer.h"', '#include ', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", @@ -98,7 +104,7 @@ def main(): "", "", "// A_pack[m_tile][K/2][32]: word[31:16]=even K, word[15:0]=odd K.", - "// vle32 with e32,m8 and VL=128 transfers four packed K-pair groups (eight registers).", + "// vle32 with e32,m4 and VL=64 transfers two packed K pairs (four registers).", f'static uint32_t Apack_dram[{K//2}*{M}] __attribute__((section(".data"), aligned(64))) =', array_to_cstr_u32(A_pack) + ";", "", diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/.gitignore b/sw/vme/apps/tew32/fp16-vq-gemm/.gitignore new file mode 100644 index 000000000..bca8a6e72 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/.gitignore @@ -0,0 +1 @@ +data/data_*.h diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/CMakeLists.txt b/sw/vme/apps/tew32/fp16-vq-gemm/CMakeLists.txt new file mode 100644 index 000000000..7564b8818 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/CMakeLists.txt @@ -0,0 +1,54 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +set(_source "${CMAKE_CURRENT_SOURCE_DIR}") + +foreach(_k IN ITEMS 128 256 512) + # Packed A, pre-expanded VQ metadata, two codebooks, and C. + math(EXPR _a_bytes "64 * ${_k} * 2") + math(EXPR _b_bytes "4 * ${_k} / 8 * 768 + 2 * 256 * 8 * 2") + math(EXPR _c_bytes "64 * 64 * 4") + math(EXPR _required_tcdm_bytes + "${_a_bytes} + ${_b_bytes} + ${_c_bytes}") + if(_required_tcdm_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping fp16-vq-gemm M64 N64 K${_k}: requires " + "${_required_tcdm_bytes} TCDM bytes, but only " + "${SNRT_TCDM_SIZE} are configured") + continue() + endif() + + set(_config "${_source}/script/gemm_64_64_${_k}.json") + set(_header "${_source}/data/data_64_64_${_k}.h") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${_source}/script/gen_data.py" + -c "${_config}" + DEPENDS "${_source}/script/gen_data.py" + "${_config}" + "${_source}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-vq-gemm-K${_k} DEPENDS "${_header}") + endif() + + foreach(_bias IN ITEMS vtle vle) + set(_target "fp16-vq-gemm-${_bias}-1x4_M64_N64_K${_k}") + if(BUILD_TESTS) + add_snitch_test(${_target} "${_source}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-vq-gemm-K${_k}) + target_include_directories(${_test_target} PRIVATE + "${_source}" "${_source}/kernel" "${_source}/data") + target_compile_options(${_test_target} PRIVATE -fno-vectorize -fno-slp-vectorize) + target_compile_definitions(${_test_target} PRIVATE + VQ_BIAS_VLE=$ + K_CHUNK=${_k} + DATAHEADER="data/data_64_64_${_k}.h") + endif() + endforeach() +endforeach() diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/README.md b/sw/vme/apps/tew32/fp16-vq-gemm/README.md new file mode 100644 index 000000000..b8c266053 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/README.md @@ -0,0 +1,50 @@ +# FP16 residual-VQ GEMM, 1x4 + +Computes `C = C_initial + A * decode(Bmeta)` with FP32 accumulation. +`C_initial` is a full row-major 64x64 FP32 bias matrix, not a broadcast +column vector. Requires TE=16, VLEN=512, M=N=64, K=128/256/512. +The fused decode and 1x4 K-loop schedule come from fp16-vq-matmul/kernel/matmul.c. +The kernel has one `gemm_fp16_fp32_vq_1x4` entry point, marked noinline +and aligned to 64 bytes. `#if VQ_BIAS_VLE` selects VLE plus VTMV when +set to 1, or VTLE when set to 0. Both builds share the same compute schedule. + +## Bias schedule + +Both versions preload all four tiles' bias before loading A and gathering B. +The first-K-group unrolling and next-tile bias overlap experiment has been +reverted. The middle K loop starts at group 0, with the final group and +VTSE drain retaining the original schedule. + +VTLE uses one FP32 matrix setup for all four tiles, then `vtle32` per row. +VLE uses the same setup followed by `vle32.v v8` and `vtmv.t.v` per row. +There are no extra VSET transitions between bias loads and VTMV batches. +Bias replaces VTZERO and is loaded once per output block. + +Bias transfers use `msetmtypei 1, 2, 0`, TN=TM=16. Before each MAC +quartet, restore `(TE << 10) | (2 << 5) | 2`, vtype=0xC8, TN=16. +TSS bases are 0, 0x20000000, 0x40000000, 0x60000000; each bias row +advances TSS by one and its memory address by 256 bytes. + +## Build and validation + +The parent apps CMakeLists registers this app. Targets are +`fp16-vq-gemm-{vtle,vle}-1x4_M64_N64_K{128,256,512}` (with the normal test +prefix). Shapes that exceed configured TCDM are skipped. Each executable +loads the same initial bias outside timing and validates its own output. +The generator seeds the FP32 reference with bias before tk=2 accumulation; +this can round differently from adding bias in an epilogue. + +The previous all-bias-preloaded K128 baseline passed RTL with zero maximum +absolute error: VTLE 29,126 cycles (14.06%), VLE 29,339 cycles (13.96%). + +The reverted overlap experiment passed K128 Questa RTL with zero errors +and zero maximum absolute error: + +| Schedule | Cycles | Utilization | Change from preload baseline | +| --- | ---: | ---: | ---: | +| VTLE unrolled / next-tile overlap | 29,145 | 14.05% | +19 | +| VLE gather-phase / four-row VTMV batches | 29,367 | 13.94% | +28 | + +Those measurements did not show a speedup; they do not isolate cache misses +from configuration or scheduling overhead. Logs for that experiment are under +`hw/system/spatz_cluster/logs/vq-overlap-k128-fcyzb9/{vtle,vle}/transcript`. diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/data/layer.h b/sw/vme/apps/tew32/fp16-vq-gemm/data/layer.h new file mode 100644 index 000000000..18bcdc3b1 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME GEMM layer configuration. + * + * Describes Cresult = C_initial + A * decode(Bidx) with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} gemm_layer; diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/kernel/gemm.c b/sw/vme/apps/tew32/fp16-vq-gemm/kernel/gemm.c new file mode 100644 index 000000000..209ffd5be --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/kernel/gemm.c @@ -0,0 +1,223 @@ +#include "gemm.h" +#define MATRIX_CONFIG \ + "msetmtype %[mt], %[vt]\n" "msettn zero, %[tn]\n" +#define MATRIX_INPUTS \ + [mt] "r"((TE << 10) | (2u << 5) | 2u), [vt] "r"(0xC8), [tn] "r"(TE) + +// Shared 1x4 schedule; only the bias transfer differs between builds. + +__attribute__((noinline, aligned(64))) +void gemm_fp16_fp32_vq_1x4(const __fp16 *Atpack, const vq_group *Bmeta, + const __fp16 *cb0, const __fp16 *cb1, float *C) { + _Static_assert(K_CHUNK % 8 == 0, "K must be a multiple of eight"); + const uint32_t *a = (const uint32_t *)Atpack; + enum { K_GROUPS = K_CHUNK / 8 }; + + for (unsigned mb = 0; mb < 4; ++mb) { + const uint32_t *ab = a + mb * TE * (K_CHUNK / 2); + const vq_group *meta0 = Bmeta; + const vq_group *meta1 = meta0 + K_GROUPS; + const vq_group *meta2 = meta1 + K_GROUPS; + const vq_group *meta3 = meta2 + K_GROUPS; + + // Seed all four tiles before preloading operands; one FP32 setup. + asm volatile("msetmtypei 1, 2, 0\n" + "msettn zero, %0\n" + "msettm zero, %0" : : "r"(TE) : "memory"); + for (unsigned tile = 0; tile < 4; ++tile) { + uintptr_t tss = tile * 0x20000000u; + const float *row_ptr = C + mb * TE * 64 + tile * TE; + for (unsigned row = 0; row < TE; ++row) { +#if VQ_BIAS_VLE + asm volatile("vle32.v v8, (%1)\n" + "vtmv.t.v %0, v8" + : : "r"(tss), "r"(row_ptr) : "v8", "memory"); +#else + asm volatile("vtle32 %0, (%1)" + : : "r"(tss), "r"(row_ptr) : "memory"); +#endif + ++tss; + row_ptr += 64; + } + } + + asm volatile( + "vsetvli zero, %[vl32], e32, m4, ta, ma\n" + "vle32.v v0, (%[ap])\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i0])\n" + "vle16.v v12, (%[j0])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vluxei16.v v20, (%[cb1]), v12\n" + : + : [ap] "r"(ab), [i0] "r"(meta0[0].off0), + [j0] "r"(meta0[0].off1), [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl32] "r"(64), [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "memory"); + + #pragma clang loop unroll(disable) + for (unsigned kg = 0; kg + 1 < K_GROUPS; ++kg) { + const vq_group *curr0 = meta0 + kg; + const vq_group *curr1 = meta1 + kg; + const vq_group *curr2 = meta2 + kg; + const vq_group *curr3 = meta3 + kg; + const vq_group *next0 = curr0 + 1; + asm volatile( + "vle16.v v8, (%[i1])\n" + "vle16.v v12, (%[j1])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s0])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i2])\n" + "vle16.v v12, (%[j2])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s1])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i3])\n" + "vle16.v v12, (%[j3])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s2])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[ni0])\n" + "vle16.v v12, (%[nj0])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s3])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt12, v0, v24\n" + "vtfmm.tvv mt12, v1, v25\n" + "vtfmm.tvv mt12, v2, v26\n" + "vtfmm.tvv mt12, v3, v27\n" + "vsetvli zero, %[vl32], e32, m4, ta, ma\n" + "vle32.v v0, (%[nap])\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + : + : [i1] "r"(curr1->off0), [j1] "r"(curr1->off1), + [i2] "r"(curr2->off0), [j2] "r"(curr2->off1), + [i3] "r"(curr3->off0), [j3] "r"(curr3->off1), + [ni0] "r"(next0->off0), [nj0] "r"(next0->off1), + [s0] "r"(curr0->scale), [s1] "r"(curr1->scale), + [s2] "r"(curr2->scale), [s3] "r"(curr3->scale), + [nap] "r"(ab + (kg + 1) * 64), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl32] "r"(64), [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "memory"); + } + + const vq_group *last0 = meta0 + K_GROUPS - 1; + const vq_group *last1 = meta1 + K_GROUPS - 1; + const vq_group *last2 = meta2 + K_GROUPS - 1; + const vq_group *last3 = meta3 + K_GROUPS - 1; + asm volatile( + "vle16.v v8, (%[i1])\n" + "vle16.v v12, (%[j1])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s0])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i2])\n" + "vle16.v v12, (%[j2])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s1])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i3])\n" + "vle16.v v12, (%[j3])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s2])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s3])\n" + "vfmul.vv v24, v20, v24\n" + MATRIX_CONFIG + "vtfmm.tvv mt12, v0, v24\n" + "vtfmm.tvv mt12, v1, v25\n" + "vtfmm.tvv mt12, v2, v26\n" + "vtfmm.tvv mt12, v3, v27\n" + : + : [i1] "r"(last1->off0), [j1] "r"(last1->off1), + [i2] "r"(last2->off0), [j2] "r"(last2->off1), + [i3] "r"(last3->off0), [j3] "r"(last3->off1), + [s0] "r"(last0->scale), [s1] "r"(last1->scale), + [s2] "r"(last2->scale), [s3] "r"(last3->scale), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "memory"); + + for (unsigned tile = 0; tile < 4; ++tile) { + uintptr_t tss = tile * 0x20000000u; + uintptr_t out = (uintptr_t)(C + mb * TE * 64 + tile * TE); + for (unsigned row = 0; row < TE; ++row) + asm volatile( + "vtse32 %0, (%1)\n" + "addi %0, %0, 1\n" + "addi %1, %1, 256\n" + : "+&r"(tss), "+&r"(out) : : "memory"); + } + } +} + diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/kernel/gemm.h b/sw/vme/apps/tew32/fp16-vq-gemm/kernel/gemm.h new file mode 100644 index 000000000..ba19c2d24 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/kernel/gemm.h @@ -0,0 +1,40 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// Pre-expanded metadata for one 16-column tile and eight K values. +// Each array is loaded as one e16,m4 vector (128 elements). +typedef struct { + uint16_t off0[128]; + uint16_t off1[128]; + uint16_t scale[128]; +} vq_group; +_Static_assert(sizeof(vq_group) == 768, "VQ metadata layout changed"); + +void gemm_fp16_fp32_vq_1x4(const __fp16 *Atpack, const vq_group *Bmeta, + const __fp16 *cb0, const __fp16 *cb1, float *C); diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/main.c b/sw/vme/apps/tew32/fp16-vq-gemm/main.c new file mode 100644 index 000000000..beb3e920d --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/main.c @@ -0,0 +1,114 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "gemm.h" +#include DATAHEADER +#include "kernel/gemm.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle) : : "memory"); + return cycle; +} + +static inline uint32_t float_bits(float value) { + union { float f; uint32_t u; } bits = {.f = value}; + return bits.u; +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + uint32_t vlenb; + asm volatile("csrr %0, vlenb" : "=r"(vlenb)); + if (vlenb != TE * sizeof(uint32_t)) { + printf("FP16 VQ GEMM requires VLEN=512 and TE=16\n"); + return 2; + } + + if (gemm_l.M != 64 || gemm_l.N != 64 || + (gemm_l.K != 128 && gemm_l.K != 256 && gemm_l.K != 512)) { + printf("Unsupported GEMM shape: expected M64_N64_K128, K256, or K512\n"); + return 2; + } + enum { + A_BYTES = sizeof(Atpack_dram), + B_BYTES = sizeof(Bmeta_dram), + CB_BYTES = sizeof(CB0_dram), + C_BYTES = sizeof(Cref_dram), + BUFFER_BYTES = A_BYTES + B_BYTES + 2 * CB_BYTES + C_BYTES + }; + _Static_assert(BUFFER_BYTES <= SNRT_TCDM_SIZE, "GEMM buffers exceed TCDM"); + uint8_t *const buffer = (uint8_t *)snrt_l1alloc(BUFFER_BYTES); + if (buffer == 0) + return 2; + __fp16 *const Atpack = (__fp16 *)buffer; + vq_group *const Bmeta = (vq_group *)(buffer + A_BYTES); + __fp16 *const cb0 = (__fp16 *)(buffer + A_BYTES + B_BYTES); + __fp16 *const cb1 = (__fp16 *)(buffer + A_BYTES + B_BYTES + CB_BYTES); + float *const C = (float *)(buffer + A_BYTES + B_BYTES + 2 * CB_BYTES); + + snrt_dma_start_1d(Atpack, Atpack_dram, A_BYTES); + snrt_dma_start_1d(Bmeta, Bmeta_dram, B_BYTES); + snrt_dma_start_1d(cb0, CB0_dram, CB_BYTES); + snrt_dma_start_1d(cb1, CB1_dram, CB_BYTES); + snrt_dma_wait_all(); + + // Reload the initial C outside the timed region for every executable. + snrt_dma_start_1d(C, Cbias_dram, C_BYTES); + snrt_dma_wait_all(); + uint32_t start = get_cycle(); + gemm_fp16_fp32_vq_1x4(Atpack, Bmeta, cb0, cb1, C); + const uint32_t cycles = get_cycle() - start; + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < gemm_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < gemm_l.N; ++col) { + const uint32_t index = row * gemm_l.N + col; + const float expected = Cref_dram[index]; + const float delta = C[index] > expected ? C[index] - expected : expected - C[index]; + const float tolerance = 0.001f + 0.001f * (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols %u", row, col); + else + printf(",%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + + printf("\n=== TEW32 FP16 VQ GEMM M%u_N%u_K%u (TE=16, 1 core) ===\n", + gemm_l.M, gemm_l.N, gemm_l.K); + printf("1x4 bias=%s Cycles: %u\n", + VQ_BIAS_VLE ? "vle" : "vtle", cycles); + printf("MaxAbsErr: 0x%08x\nErrors: %u\nStatus: %s\n", + float_bits(max_abs), errors, errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_128.json b/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_128.json new file mode 100644 index 000000000..f35ec0e9d --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_128.json @@ -0,0 +1,9 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_256.json b/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_256.json new file mode 100644 index 000000000..fec92ef06 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_256.json @@ -0,0 +1,9 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_512.json b/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_512.json new file mode 100644 index 000000000..266175b16 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/script/gemm_64_64_512.json @@ -0,0 +1,9 @@ +{ + kernel: "GEMM", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-vq-gemm/script/gen_data.py b/sw/vme/apps/tew32/fp16-vq-gemm/script/gen_data.py new file mode 100644 index 000000000..1ab149a64 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-gemm/script/gen_data.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 +"""Generate D=8 residual-VQ B and a packed-A FP16-to-FP32 GEMM reference.""" +import argparse +from pathlib import Path + +import hjson +import numpy as np + + +def initializer(array, dtype): + if dtype == "float": + return "{" + ", ".join(repr(float(x)) + "f" for x in array.flat) + "}" + return "{" + ", ".join(hex(int(x)) for x in array.flat) + "}" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("-c", "--cfg", type=Path, required=True) + args = parser.parse_args() + cfg = hjson.loads(args.cfg.read_text()) + m, n, k = (int(cfg[x]) for x in ("M", "N", "K")) + if (m, n) != (64, 64) or k not in (128, 256, 512): + parser.error("expected M64 N64 and K128, K256, or K512") + if (int(cfg["TM"]), int(cfg["TN"])) != (16, 16): + parser.error("expected TE=16") + + rng = np.random.default_rng(int(cfg.get("seed", 42))) + a = rng.normal(size=(m, k)).astype(np.float16) + cb0 = rng.normal(scale=0.5, size=(256, 8)).astype(np.float16) + cb1 = rng.normal(scale=0.25, size=(256, 8)).astype(np.float16) + idx0 = rng.integers(0, 256, size=(k, n // 8), dtype=np.uint8) + idx1 = rng.integers(0, 256, size=(k, n // 8), dtype=np.uint8) + scales = rng.uniform(0.5, 1.5, size=k).astype(np.float16) + # Match the separate FP16 RVV vfadd and vfmul rounding. + summed = (cb0[idx0].astype(np.float32) + cb1[idx1].astype(np.float32)).astype(np.float16) + b = (summed.astype(np.float32) * scales[:, None, None].astype(np.float32)).astype(np.float16).reshape(k, n) + # Bias seeds the accumulator before the first dot product. + bias = rng.normal(size=(m, n)).astype(np.float32) + reference = bias.copy() + for q in range(0, k, 2): + pair = (a[:, q, None].astype(np.float64) * b[None, q, :].astype(np.float64) + + a[:, q + 1, None].astype(np.float64) * b[None, q + 1, :].astype(np.float64)) + reference = (reference.astype(np.float64) + pair).astype(np.float32) + abits = a.view(np.uint16).astype(np.uint32) + awords = (abits[:, 0::2] << 16) | abits[:, 1::2] + apack = awords.reshape(m // 16, 16, k // 2).transpose(0, 2, 1).copy() + # Pre-expand the indexed-load byte offsets and replicated scales in the + # exact e16,m4 lane order consumed by each eight-K kernel iteration. + meta = np.empty((n // 16, k // 8, 3, 128), dtype=np.uint16) + scale_bits = scales.view(np.uint16) + for nt in range(n // 16): + for kg in range(k // 8): + off0, off1, scale = meta[nt, kg] + for pair in range(4): + even_k = kg * 8 + pair * 2 + odd_k = even_k + 1 + for col in range(16): + lane = col % 8 + group = nt * 2 + col // 8 + j = pair * 32 + 2 * col + off0[j] = 2 * (8 * int(idx0[odd_k, group]) + lane) + off1[j] = 2 * (8 * int(idx1[odd_k, group]) + lane) + scale[j] = scale_bits[odd_k] + off0[j + 1] = 2 * (8 * int(idx0[even_k, group]) + lane) + off1[j + 1] = 2 * (8 * int(idx1[even_k, group]) + lane) + scale[j + 1] = scale_bits[even_k] + + lines = ["// Generated by script/gen_data.py; do not edit.", + '#include "layer.h"', '#include "gemm.h"', + f"const gemm_layer gemm_l = {{.M={m}, .N={n}, .K={k}}};"] + for name, values, dtype in ( + ("Atpack_dram", apack, "uint32_t"), + ("CB0_dram", cb0.view(np.uint16), "uint16_t"), + ("CB1_dram", cb1.view(np.uint16), "uint16_t"), + ("Cref_dram", reference, "float"), + ("Cbias_dram", bias, "float"), + ): + lines.append(f'static {dtype} {name}[{values.size}] ' + '__attribute__((section(".data"), aligned(64))) = ' + + initializer(values, dtype) + ";") + records = [] + for group in meta.reshape(-1, 3, 128): + records.append(" {" + ", ".join(initializer(x, "uint16_t") + for x in group) + "}") + lines.append(f'static vq_group Bmeta_dram[{len(records)}] ' + '__attribute__((section(".data"), aligned(64))) = {\n' + + ",\n".join(records) + "\n};") + output = Path(__file__).resolve().parent.parent / "data" / f"data_{m}_{n}_{k}.h" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("\n\n".join(lines) + "\n") + print(f"Written: {output}") + + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/.gitignore b/sw/vme/apps/tew32/fp16-vq-matmul/.gitignore new file mode 100644 index 000000000..8bec9d59a --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/.gitignore @@ -0,0 +1 @@ +/data/data_*.h diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/CMakeLists.txt b/sw/vme/apps/tew32/fp16-vq-matmul/CMakeLists.txt new file mode 100644 index 000000000..e8f143a49 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/CMakeLists.txt @@ -0,0 +1,48 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Python3 COMPONENTS Interpreter REQUIRED) + +set(_source "${CMAKE_CURRENT_SOURCE_DIR}") + +foreach(_k IN ITEMS 128 256 512) + # Packed A, pre-expanded VQ metadata, two codebooks, and C. + math(EXPR _a_bytes "64 * ${_k} * 2") + math(EXPR _b_bytes "4 * ${_k} / 8 * 768 + 2 * 256 * 8 * 2") + math(EXPR _c_bytes "64 * 64 * 4") + math(EXPR _required_tcdm_bytes + "${_a_bytes} + ${_b_bytes} + ${_c_bytes}") + if(_required_tcdm_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping fp16-vq-matmul M64 N64 K${_k}: requires " + "${_required_tcdm_bytes} TCDM bytes, but only " + "${SNRT_TCDM_SIZE} are configured") + continue() + endif() + + set(_target "fp16-vq-matmul_M64_N64_K${_k}") + set(_config "${_source}/script/matmul_64_64_${_k}.json") + set(_header "${_source}/data/data_64_64_${_k}.h") + + add_custom_command( + OUTPUT "${_header}" + COMMAND ${Python3_EXECUTABLE} "${_source}/script/gen_data.py" + -c "${_config}" + DEPENDS "${_source}/script/gen_data.py" + "${_config}" + "${_source}/data/layer.h" + VERBATIM) + + if(BUILD_TESTS) + add_custom_target(generate-${_target} DEPENDS "${_header}") + add_snitch_test(${_target} "${_source}/main.c") + set(_test_target "test-${SNITCH_TEST_PREFIX}${_target}") + add_dependencies(${_test_target} generate-${_target}) + target_include_directories(${_test_target} PRIVATE + "${_source}" "${_source}/kernel" "${_source}/data") + target_compile_options(${_test_target} PRIVATE -fno-vectorize -fno-slp-vectorize) + target_compile_definitions(${_test_target} PRIVATE + K_CHUNK=${_k} + DATAHEADER="data/data_64_64_${_k}.h") + endif() +endforeach() diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/README.md b/sw/vme/apps/tew32/fp16-vq-matmul/README.md new file mode 100644 index 000000000..d10c2c45d --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/README.md @@ -0,0 +1,44 @@ +# FP16 residual-VQ MatMul + +Computes FP32 C = A * decode(Bidx), without a bias addend. +Requires TE=16, VLEN=512, M=N=64 and K=128/256/512. +No instruction-buffer setting is changed by this app. + +## Preprocessing + +Two 256x8 FP16 codebooks are added and scaled, with FP16 rounding after +each operation. Standard RVV indexed loads, addition and multiplication +use e16,m4, VL=128. Scalar code expands the compressed indices to byte +offsets. Decoded B is materialized in TCDM once and reused across M blocks; +this is not a fused decode/compute pipeline. Timing includes this decode. +Packing is even K in the upper halfword and odd K in the lower halfword. + +## Compute schedule + +The four tiles cover a 16x64 output block. Grouped operand loads use +e32,m4, VL=64; every load phase restores msetmtype and msettn before +matrix operations. Matrix compute uses e16, tk=2, FP32 accumulation. +vtse32 follows the reference FP16 VTSE kernel's matrix configuration. + +The middle loop handles K-16 elements. The final 16 K elements are written +explicitly as eight tk=2 vtfmm instructions per tile: + +- Finish mt0. +- Compute mt4 while storing all 16 rows of mt0. +- Compute mt8 while storing all 16 rows of mt4. +- Compute mt12 while storing all 16 rows of mt8. +- Store mt12 during the next M block's first four mt0 instructions. + Only then clear mt12 for reuse. Skip the already computed mt0 K group. +- Drain the last block's mt12 separately. + +The tail retains A in v0-v3/v8-v11 and B in v16-v19/v24-v27. +The companion registers used implicitly by tk=2 remain available. +Each vtse advances its destination by 256 bytes and its TSS by one. + +## Validation + +Run script/check_schedule.py for an independent host model of decode +packing, K coverage and output-address coverage. This is not an RTL test. +CMake skips shapes whose buffers do not fit SNRT_TCDM_SIZE. Buffers are +single-resident, not double buffered; the budget includes packed B and +768 bytes of decode scratch, but does not represent total runtime usage. diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/data/layer.h b/sw/vme/apps/tew32/fp16-vq-matmul/data/layer.h new file mode 100644 index 000000000..741b6b790 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME MatMul layer configuration. + * + * Describes Cresult = A * decode(Bidx) with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/kernel/matmul.c b/sw/vme/apps/tew32/fp16-vq-matmul/kernel/matmul.c new file mode 100644 index 000000000..fc2bfc5e9 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/kernel/matmul.c @@ -0,0 +1,340 @@ +#include "matmul.h" +#define MATRIX_CONFIG \ + "msetmtype %[mt], %[vt]\n" "msettn zero, %[tn]\n" +#define MATRIX_INPUTS \ + [mt] "r"((TE << 10) | (2u << 5) | 2u), [vt] "r"(0xC8), [tn] "r"(TE) + +__attribute__((noinline, aligned(64))) +void matmul_fp16_fp32_vq_1x2(const __fp16 *Atpack, const vq_group *Bmeta, + const __fp16 *cb0, const __fp16 *cb1, float *C) { + _Static_assert(K_CHUNK % 8 == 0, "K must be a multiple of eight"); + const uint32_t *a = (const uint32_t *)Atpack; + enum { K_GROUPS = K_CHUNK / 8 }; + + for (unsigned mb = 0; mb < 4; ++mb) { + const uint32_t *ab = a + mb * TE * (K_CHUNK / 2); + for (unsigned nb = 0; nb < 4; nb += 2) { + const vq_group *meta0 = Bmeta + nb * K_GROUPS; + const vq_group *meta1 = meta0 + K_GROUPS; + + // Preload A and the two raw codebook vectors for mt0. + asm volatile( + MATRIX_CONFIG + "vtzero mt0\n" + "vtzero mt4\n" + "vsetvli zero, %[vl32], e32, m4, ta, ma\n" + "vle32.v v0, (%[ap])\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i0])\n" + "vle16.v v12, (%[j0])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vluxei16.v v20, (%[cb1]), v12\n" + : + : [ap] "r"(ab), [i0] "r"(meta0[0].off0), + [j0] "r"(meta0[0].off1), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl32] "r"(64), [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "memory"); + + #pragma clang loop unroll(disable) + for (unsigned kg = 0; kg + 1 < K_GROUPS; ++kg) { + const vq_group *next0 = meta0 + kg + 1; + const vq_group *curr1 = meta1 + kg; + const vq_group *curr0 = meta0 + kg; + asm volatile( + // Finish mt0 preprocessing while preloading mt4 indices. + "vle16.v v8, (%[i1])\n" + "vle16.v v12, (%[j1])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s0])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + // Finish mt4 preprocessing while gathering the next mt0. + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[ni0])\n" + "vle16.v v12, (%[nj0])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s1])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + // The next raw mt0 vectors are resident; preload only A. + "vsetvli zero, %[vl32], e32, m4, ta, ma\n" + "vle32.v v0, (%[nap])\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + : + : [i1] "r"(curr1->off0), [j1] "r"(curr1->off1), + [s1] "r"(curr1->scale), [ni0] "r"(next0->off0), + [nj0] "r"(next0->off1), [s0] "r"(curr0->scale), + [nap] "r"(ab + (kg + 1) * 64), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl32] "r"(64), [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "memory"); + } + + // Drain the final raw mt0 group and decode/compute mt4. + const vq_group *last0 = meta0 + K_GROUPS - 1; + const vq_group *last1 = meta1 + K_GROUPS - 1; + asm volatile( + "vle16.v v8, (%[i1])\n" + "vle16.v v12, (%[j1])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s0])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s1])\n" + "vfmul.vv v24, v20, v24\n" + MATRIX_CONFIG + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + : + : [i1] "r"(last1->off0), [j1] "r"(last1->off1), + [s0] "r"(last0->scale), [s1] "r"(last1->scale), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "memory"); + + for (unsigned tile = 0; tile < 2; ++tile) { + uintptr_t tss = tile * 0x20000000u; + uintptr_t out = (uintptr_t)(C + mb * TE * 64 + (nb + tile) * TE); + for (unsigned row = 0; row < TE; ++row) + asm volatile( + "vtse32 %0, (%1)\n" + "addi %0, %0, 1\n" + "addi %1, %1, 256\n" + : "+&r"(tss), "+&r"(out) : : "memory"); + } + } + } +} + +__attribute__((noinline, aligned(64))) +void matmul_fp16_fp32_vq_1x4(const __fp16 *Atpack, const vq_group *Bmeta, + const __fp16 *cb0, const __fp16 *cb1, float *C) { + _Static_assert(K_CHUNK % 8 == 0, "K must be a multiple of eight"); + const uint32_t *a = (const uint32_t *)Atpack; + enum { K_GROUPS = K_CHUNK / 8 }; + + for (unsigned mb = 0; mb < 4; ++mb) { + const uint32_t *ab = a + mb * TE * (K_CHUNK / 2); + const vq_group *meta0 = Bmeta; + const vq_group *meta1 = meta0 + K_GROUPS; + const vq_group *meta2 = meta1 + K_GROUPS; + const vq_group *meta3 = meta2 + K_GROUPS; + + asm volatile( + MATRIX_CONFIG + "vtzero mt0\n" + "vtzero mt4\n" + "vtzero mt8\n" + "vtzero mt12\n" + "vsetvli zero, %[vl32], e32, m4, ta, ma\n" + "vle32.v v0, (%[ap])\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i0])\n" + "vle16.v v12, (%[j0])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vluxei16.v v20, (%[cb1]), v12\n" + : + : [ap] "r"(ab), [i0] "r"(meta0[0].off0), + [j0] "r"(meta0[0].off1), [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl32] "r"(64), [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v8", "v9", "v10", "v11", + "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", + "v20", "v21", "v22", "v23", "memory"); + + #pragma clang loop unroll(disable) + for (unsigned kg = 0; kg + 1 < K_GROUPS; ++kg) { + const vq_group *curr0 = meta0 + kg; + const vq_group *curr1 = meta1 + kg; + const vq_group *curr2 = meta2 + kg; + const vq_group *curr3 = meta3 + kg; + const vq_group *next0 = curr0 + 1; + asm volatile( + "vle16.v v8, (%[i1])\n" + "vle16.v v12, (%[j1])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s0])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i2])\n" + "vle16.v v12, (%[j2])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s1])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i3])\n" + "vle16.v v12, (%[j3])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s2])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[ni0])\n" + "vle16.v v12, (%[nj0])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s3])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt12, v0, v24\n" + "vtfmm.tvv mt12, v1, v25\n" + "vtfmm.tvv mt12, v2, v26\n" + "vtfmm.tvv mt12, v3, v27\n" + "vsetvli zero, %[vl32], e32, m4, ta, ma\n" + "vle32.v v0, (%[nap])\n" + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + : + : [i1] "r"(curr1->off0), [j1] "r"(curr1->off1), + [i2] "r"(curr2->off0), [j2] "r"(curr2->off1), + [i3] "r"(curr3->off0), [j3] "r"(curr3->off1), + [ni0] "r"(next0->off0), [nj0] "r"(next0->off1), + [s0] "r"(curr0->scale), [s1] "r"(curr1->scale), + [s2] "r"(curr2->scale), [s3] "r"(curr3->scale), + [nap] "r"(ab + (kg + 1) * 64), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl32] "r"(64), [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "memory"); + } + + const vq_group *last0 = meta0 + K_GROUPS - 1; + const vq_group *last1 = meta1 + K_GROUPS - 1; + const vq_group *last2 = meta2 + K_GROUPS - 1; + const vq_group *last3 = meta3 + K_GROUPS - 1; + asm volatile( + "vle16.v v8, (%[i1])\n" + "vle16.v v12, (%[j1])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s0])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i2])\n" + "vle16.v v12, (%[j2])\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s1])\n" + "vluxei16.v v16, (%[cb0]), v8\n" + "vfmul.vv v24, v20, v24\n" + "vluxei16.v v20, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt4, v0, v24\n" + "vtfmm.tvv mt4, v1, v25\n" + "vtfmm.tvv mt4, v2, v26\n" + "vtfmm.tvv mt4, v3, v27\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vle16.v v8, (%[i3])\n" + "vle16.v v12, (%[j3])\n" + "vfadd.vv v16, v16, v20\n" + "vle16.v v28, (%[s2])\n" + "vluxei16.v v24, (%[cb0]), v8\n" + "vfmul.vv v16, v28, v16\n" + "vluxei16.v v28, (%[cb1]), v12\n" + MATRIX_CONFIG + "vtfmm.tvv mt8, v0, v16\n" + "vtfmm.tvv mt8, v1, v17\n" + "vtfmm.tvv mt8, v2, v18\n" + "vtfmm.tvv mt8, v3, v19\n" + + "vsetvli zero, %[vl16], e16, m4, ta, ma\n" + "vfadd.vv v24, v24, v28\n" + "vle16.v v20, (%[s3])\n" + "vfmul.vv v24, v20, v24\n" + MATRIX_CONFIG + "vtfmm.tvv mt12, v0, v24\n" + "vtfmm.tvv mt12, v1, v25\n" + "vtfmm.tvv mt12, v2, v26\n" + "vtfmm.tvv mt12, v3, v27\n" + : + : [i1] "r"(last1->off0), [j1] "r"(last1->off1), + [i2] "r"(last2->off0), [j2] "r"(last2->off1), + [i3] "r"(last3->off0), [j3] "r"(last3->off1), + [s0] "r"(last0->scale), [s1] "r"(last1->scale), + [s2] "r"(last2->scale), [s3] "r"(last3->scale), + [cb0] "r"(cb0), [cb1] "r"(cb1), + [vl16] "r"(128), MATRIX_INPUTS + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", + "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", + "memory"); + + for (unsigned tile = 0; tile < 4; ++tile) { + uintptr_t tss = tile * 0x20000000u; + uintptr_t out = (uintptr_t)(C + mb * TE * 64 + tile * TE); + for (unsigned row = 0; row < TE; ++row) + asm volatile( + "vtse32 %0, (%1)\n" + "addi %0, %0, 1\n" + "addi %1, %1, 256\n" + : "+&r"(tss), "+&r"(out) : : "memory"); + } + } +} + diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/kernel/matmul.h b/sw/vme/apps/tew32/fp16-vq-matmul/kernel/matmul.h new file mode 100644 index 000000000..be07d53a8 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/kernel/matmul.h @@ -0,0 +1,42 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, +}; + +// Pre-expanded metadata for one 16-column tile and eight K values. +// Each array is loaded as one e16,m4 vector (128 elements). +typedef struct { + uint16_t off0[128]; + uint16_t off1[128]; + uint16_t scale[128]; +} vq_group; +_Static_assert(sizeof(vq_group) == 768, "VQ metadata layout changed"); + +void matmul_fp16_fp32_vq_1x2(const __fp16 *Atpack, const vq_group *Bmeta, + const __fp16 *cb0, const __fp16 *cb1, float *C); +void matmul_fp16_fp32_vq_1x4(const __fp16 *Atpack, const vq_group *Bmeta, + const __fp16 *cb0, const __fp16 *cb1, float *C); diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/main.c b/sw/vme/apps/tew32/fp16-vq-matmul/main.c new file mode 100644 index 000000000..68a991d8b --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/main.c @@ -0,0 +1,118 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include +#include + +#include "matmul.h" +#include DATAHEADER +#include "kernel/matmul.c" + +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle) : : "memory"); + return cycle; +} + +static inline uint32_t float_bits(float value) { + union { float f; uint32_t u; } bits = {.f = value}; + return bits.u; +} + +int main(void) { + if (snrt_cluster_core_idx() != 0) + return 0; + + uint32_t vlenb; + asm volatile("csrr %0, vlenb" : "=r"(vlenb)); + if (vlenb != TE * sizeof(uint32_t)) { + printf("FP16 VQ MatMul requires VLEN=512 and TE=16\n"); + return 2; + } + + if (matmul_l.M != 64 || matmul_l.N != 64 || + (matmul_l.K != 128 && matmul_l.K != 256 && matmul_l.K != 512)) { + printf("Unsupported MatMul shape: expected M64_N64_K128, K256, or K512\n"); + return 2; + } + enum { + A_BYTES = sizeof(Atpack_dram), + B_BYTES = sizeof(Bmeta_dram), + CB_BYTES = sizeof(CB0_dram), + C_BYTES = sizeof(Cref_dram), + BUFFER_BYTES = A_BYTES + B_BYTES + 2 * CB_BYTES + C_BYTES + }; + _Static_assert(BUFFER_BYTES <= SNRT_TCDM_SIZE, "MatMul buffers exceed TCDM"); + uint8_t *const buffer = (uint8_t *)snrt_l1alloc(BUFFER_BYTES); + if (buffer == 0) + return 2; + __fp16 *const Atpack = (__fp16 *)buffer; + vq_group *const Bmeta = (vq_group *)(buffer + A_BYTES); + __fp16 *const cb0 = (__fp16 *)(buffer + A_BYTES + B_BYTES); + __fp16 *const cb1 = (__fp16 *)(buffer + A_BYTES + B_BYTES + CB_BYTES); + float *const C = (float *)(buffer + A_BYTES + B_BYTES + 2 * CB_BYTES); + + snrt_dma_start_1d(Atpack, Atpack_dram, A_BYTES); + snrt_dma_start_1d(Bmeta, Bmeta_dram, B_BYTES); + snrt_dma_start_1d(cb0, CB0_dram, CB_BYTES); + snrt_dma_start_1d(cb1, CB1_dram, CB_BYTES); + snrt_dma_wait_all(); + + uint32_t start = get_cycle(); + matmul_fp16_fp32_vq_1x2(Atpack, Bmeta, cb0, cb1, C); + const uint32_t cycles_1x2 = get_cycle() - start; + + start = get_cycle(); + matmul_fp16_fp32_vq_1x4(Atpack, Bmeta, cb0, cb1, C); + const uint32_t cycles_1x4 = get_cycle() - start; + + uint32_t errors = 0; + float max_abs = 0.0f; + for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; + for (uint32_t col = 0; col < matmul_l.N; ++col) { + const uint32_t index = row * matmul_l.N + col; + const float expected = Cref_dram[index]; + const float delta = C[index] > expected ? C[index] - expected : expected - C[index]; + const float tolerance = 0.001f + 0.001f * (expected < 0.0f ? -expected : expected); + if (delta > max_abs) + max_abs = delta; + if (!(delta <= tolerance)) { + if (row_errors == 0) + printf("row %u: errors at cols %u", row, col); + else + printf(",%u", col); + ++row_errors; + ++errors; + } + } + if (row_errors != 0) + printf(" (%u errors)\n", row_errors); + } + const uint64_t useful_macs = (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; + const uint64_t peak_macs = (uint64_t)cycles_1x4 * CE * CE * 2; + const uint32_t utilization = peak_macs ? useful_macs * 10000 / peak_macs : 0; + printf("\n=== TEW32 FP16 VQ MatMul M%u_N%u_K%u (TE=16, 1 core) ===\n", + matmul_l.M, matmul_l.N, matmul_l.K); + printf("1x2 Cycles: %u\n", cycles_1x2); + printf("1x4 Cycles: %u\n", cycles_1x4); + printf("1x4 Utilization: %u.%02u%%\n", utilization / 100, utilization % 100); + printf("MaxAbsErr: 0x%08x\nErrors: %u\nStatus: %s\n", + float_bits(max_abs), errors, errors ? "FAIL" : "PASS"); + return errors ? 1 : 0; +} diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/script/check_schedule.py b/sw/vme/apps/tew32/fp16-vq-matmul/script/check_schedule.py new file mode 100644 index 000000000..a8d86a6e0 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/script/check_schedule.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Host-side address/coverage model; not a replacement for RTL simulation.""" +import numpy as np + + +def check(k): + rng = np.random.default_rng(42) + cb0 = rng.normal(size=(256, 8)).astype(np.float16) + cb1 = rng.normal(size=(256, 8)).astype(np.float16) + ids0 = rng.integers(256, size=(k, 8)) + ids1 = rng.integers(256, size=(k, 8)) + scale = rng.uniform(.5, 1.5, size=k).astype(np.float16) + expected = ((cb0[ids0].astype(np.float32) + + cb1[ids1].astype(np.float32)).astype(np.float16).astype(np.float32) * + scale[:, None, None].astype(np.float32)).astype(np.float16).reshape(k, 64) + packed = np.empty((4, k // 2, 16), dtype=np.uint32) + for tile in range(4): + for pair in range(k // 2): + for col in range(16): + g, lane = tile * 2 + col // 8, col % 8 + bits = [] + for q in (2 * pair + 1, 2 * pair): + o0 = 2 * (8 * ids0[q, g] + lane) + o1 = 2 * (8 * ids1[q, g] + lane) + value = np.float16(np.float32(cb0.flat[o0 // 2]) + + np.float32(cb1.flat[o1 // 2])) + value = np.float16(np.float32(value) * np.float32(scale[q])) + bits.append(int(value.view(np.uint16))) + packed[tile, pair, col] = bits[0] | bits[1] << 16 + reference_bits = expected.view(np.uint16).astype(np.uint32) + for tile in range(4): + cols = slice(tile * 16, (tile + 1) * 16) + assert np.array_equal(packed[tile], + (reference_bits[0::2, cols] << 16) | + reference_bits[1::2, cols]) + + covered = np.zeros((4, 4, k // 2), dtype=int) + stores = np.zeros((64, 64), dtype=int) + pending = None + + def store_tile(block, tile): + assert np.all(covered[block, tile] == 1), "store before completion" + for row in range(16): + # Actual byte address: base + 256 per stored tile row. + addr = ((block * 16) * 64 + tile * 16) * 4 + row * 256 + r, c = divmod(addr // 4, 64) + stores[r, c:c + 16] += 1 + + for block in range(4): + if block: + covered[block, 0, :4] += 1 + assert pending == block - 1 + store_tile(pending, 3) + pending = None # mt12 can now be zeroed. + for kg in range(k // 8 - 2): + for tile in range(4): + if block and kg == 0 and tile == 0: + continue + covered[block, tile, kg * 4:kg * 4 + 4] += 1 + for tile in range(4): + covered[block, tile, k // 2 - 8:] += 1 + if tile: + store_tile(block, tile - 1) + pending = block + store_tile(pending, 3) + assert np.all(covered == 1), "missing or duplicate K pairs" + assert np.all(stores == 1), "missing or duplicate output elements" + print(f"K{k}: decode packing, K coverage, store addresses: PASS") + + +if __name__ == "__main__": + for k in (128, 256, 512): + check(k) diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/script/gen_data.py b/sw/vme/apps/tew32/fp16-vq-matmul/script/gen_data.py new file mode 100644 index 000000000..a6c7fd30e --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/script/gen_data.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright 2026 ETH Zurich and University of Bologna. +# SPDX-License-Identifier: Apache-2.0 +"""Generate D=8 residual-VQ B and a packed-A FP16-to-FP32 MatMul reference.""" +import argparse +from pathlib import Path + +import hjson +import numpy as np + + +def initializer(array, dtype): + if dtype == "float": + return "{" + ", ".join(repr(float(x)) + "f" for x in array.flat) + "}" + return "{" + ", ".join(hex(int(x)) for x in array.flat) + "}" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("-c", "--cfg", type=Path, required=True) + args = parser.parse_args() + cfg = hjson.loads(args.cfg.read_text()) + m, n, k = (int(cfg[x]) for x in ("M", "N", "K")) + if (m, n) != (64, 64) or k not in (128, 256, 512): + parser.error("expected M64 N64 and K128, K256, or K512") + if (int(cfg["TM"]), int(cfg["TN"])) != (16, 16): + parser.error("expected TE=16") + + rng = np.random.default_rng(int(cfg.get("seed", 42))) + a = rng.normal(size=(m, k)).astype(np.float16) + cb0 = rng.normal(scale=0.5, size=(256, 8)).astype(np.float16) + cb1 = rng.normal(scale=0.25, size=(256, 8)).astype(np.float16) + idx0 = rng.integers(0, 256, size=(k, n // 8), dtype=np.uint8) + idx1 = rng.integers(0, 256, size=(k, n // 8), dtype=np.uint8) + scales = rng.uniform(0.5, 1.5, size=k).astype(np.float16) + # Match the separate FP16 RVV vfadd and vfmul rounding. + summed = (cb0[idx0].astype(np.float32) + cb1[idx1].astype(np.float32)).astype(np.float16) + b = (summed.astype(np.float32) * scales[:, None, None].astype(np.float32)).astype(np.float16).reshape(k, n) + # Tiles start at zero; this MatMul has no bias epilogue. + reference = np.zeros((m, n), dtype=np.float32) + for q in range(0, k, 2): + pair = (a[:, q, None].astype(np.float64) * b[None, q, :].astype(np.float64) + + a[:, q + 1, None].astype(np.float64) * b[None, q + 1, :].astype(np.float64)) + reference = (reference.astype(np.float64) + pair).astype(np.float32) + abits = a.view(np.uint16).astype(np.uint32) + awords = (abits[:, 0::2] << 16) | abits[:, 1::2] + apack = awords.reshape(m // 16, 16, k // 2).transpose(0, 2, 1).copy() + # Pre-expand the indexed-load byte offsets and replicated scales in the + # exact e16,m4 lane order consumed by each eight-K kernel iteration. + meta = np.empty((n // 16, k // 8, 3, 128), dtype=np.uint16) + scale_bits = scales.view(np.uint16) + for nt in range(n // 16): + for kg in range(k // 8): + off0, off1, scale = meta[nt, kg] + for pair in range(4): + even_k = kg * 8 + pair * 2 + odd_k = even_k + 1 + for col in range(16): + lane = col % 8 + group = nt * 2 + col // 8 + j = pair * 32 + 2 * col + off0[j] = 2 * (8 * int(idx0[odd_k, group]) + lane) + off1[j] = 2 * (8 * int(idx1[odd_k, group]) + lane) + scale[j] = scale_bits[odd_k] + off0[j + 1] = 2 * (8 * int(idx0[even_k, group]) + lane) + off1[j + 1] = 2 * (8 * int(idx1[even_k, group]) + lane) + scale[j + 1] = scale_bits[even_k] + + lines = ["// Generated by script/gen_data.py; do not edit.", + '#include "layer.h"', '#include "matmul.h"', + f"const matmul_layer matmul_l = {{.M={m}, .N={n}, .K={k}}};"] + for name, values, dtype in ( + ("Atpack_dram", apack, "uint32_t"), + ("CB0_dram", cb0.view(np.uint16), "uint16_t"), + ("CB1_dram", cb1.view(np.uint16), "uint16_t"), + ("Cref_dram", reference, "float"), + ): + lines.append(f'static {dtype} {name}[{values.size}] ' + '__attribute__((section(".data"), aligned(64))) = ' + + initializer(values, dtype) + ";") + records = [] + for group in meta.reshape(-1, 3, 128): + records.append(" {" + ", ".join(initializer(x, "uint16_t") + for x in group) + "}") + lines.append(f'static vq_group Bmeta_dram[{len(records)}] ' + '__attribute__((section(".data"), aligned(64))) = {\n' + + ",\n".join(records) + "\n};") + output = Path(__file__).resolve().parent.parent / "data" / f"data_{m}_{n}_{k}.h" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text("\n\n".join(lines) + "\n") + print(f"Written: {output}") + + +if __name__ == "__main__": + main() diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_128.json b/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_128.json new file mode 100644 index 000000000..33246f663 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_128.json @@ -0,0 +1,9 @@ +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 128, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_256.json b/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_256.json new file mode 100644 index 000000000..d52fbff52 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_256.json @@ -0,0 +1,9 @@ +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 256, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_512.json b/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_512.json new file mode 100644 index 000000000..4cdf55f16 --- /dev/null +++ b/sw/vme/apps/tew32/fp16-vq-matmul/script/matmul_64_64_512.json @@ -0,0 +1,9 @@ +{ + kernel: "MatMul", + M: 64, + N: 64, + K: 512, + TM: 16, + TN: 16, + seed: 42 +} diff --git a/sw/vme/apps/tew32/fp32-gemm-vle-vse/CMakeLists.txt b/sw/vme/apps/tew32/fp32-gemm-vle-vse/CMakeLists.txt deleted file mode 100644 index 836389ddc..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vle-vse/CMakeLists.txt +++ /dev/null @@ -1,69 +0,0 @@ - -option(VME_GEMM_SINGLE_CORE - "Run general VME GEMM kernels on one compute core" OFF) - -set(VME_GEMM_SHAPES - "64x64x64;64x64x128;64x128x64;64x128x128;128x64x64;128x64x128;128x128x64;128x128x128" CACHE STRING - "Semicolon-separated MxNxK shapes for the general FP32 GEMM tests") - -set(_vme_gemm_build_shapes ${VME_GEMM_SHAPES}) -list(APPEND _vme_gemm_build_shapes - "13x13x13" "13x23x13" "23x13x13" "23x23x13" - "13x45x7" "13x55x7" "23x45x7" "23x55x7" - "45x13x8" "45x23x8" "55x13x8" "55x23x8" - "45x45x9" "45x55x9" "55x45x9" "55x55x9" - "77x13x13" "77x23x13" "87x13x13" "87x23x13" - "77x45x9" "77x55x9" "87x45x9" "87x55x9" - "77x77x77" "77x87x77" "87x77x77" "87x87x77" - "77x109x71" "77x119x71" "87x109x71" "87x119x71" - "109x77x72" "109x87x72" "119x77x72" "119x87x72" - "109x109x73" "109x119x73" "119x109x73" "119x119x73" - "141x77x77" "141x87x77" "151x77x77" "151x87x77" - "141x109x73" "141x119x73" "151x109x73" "151x119x73" - "17x53x95" "64x64x256" "64x96x128" "64x128x192" - "724x256x192") -set(_vme_gemm_normalized_shapes) -foreach(_shape IN LISTS _vme_gemm_build_shapes) - string(STRIP "${_shape}" _shape) - list(APPEND _vme_gemm_normalized_shapes "${_shape}") -endforeach() -set(_vme_gemm_build_shapes ${_vme_gemm_normalized_shapes}) -list(REMOVE_DUPLICATES _vme_gemm_build_shapes) - -foreach(_shape IN LISTS _vme_gemm_build_shapes) - if(NOT _shape MATCHES "^([0-9]+)x([0-9]+)x([0-9]+)$") - message(FATAL_ERROR "Invalid GEMM shape '${_shape}', expected MxNxK") - endif() - set(_m ${CMAKE_MATCH_1}) - set(_n ${CMAKE_MATCH_2}) - set(_k ${CMAKE_MATCH_3}) - set(_target fp32-gemm-vle-vse_M${_m}_N${_n}_K${_k}) - set(_generated_dir ${CMAKE_CURRENT_BINARY_DIR}/generated) - set(_header ${_generated_dir}/data/data_${_m}_${_n}_${_k}.h) - - add_custom_command( - OUTPUT ${_header} - COMMAND ${CMAKE_COMMAND} -E make_directory ${_generated_dir}/data - COMMAND ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py - --m ${_m} --n ${_n} --k ${_k} --output ${_header} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py - ${CMAKE_CURRENT_SOURCE_DIR}/data/layer.h - VERBATIM) - - if(BUILD_TESTS) - add_custom_target(generate-${_target} DEPENDS ${_header}) - add_snitch_test(${_target} main.c) - add_dependencies(test-${SNITCH_TEST_PREFIX}${_target} generate-${_target}) - target_include_directories(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/data - ${_generated_dir}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - DATAHEADER="data/data_${_m}_${_n}_${_k}.h" - KERNELHEADER="kernel/gemm.c") - if(VME_GEMM_SINGLE_CORE) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - SPATZ_GEMM_SINGLE_CORE=1) - endif() - endif() -endforeach() diff --git a/sw/vme/apps/tew32/fp32-gemm-vle-vse/data/layer.h b/sw/vme/apps/tew32/fp32-gemm-vle-vse/data/layer.h deleted file mode 100644 index 76e859a50..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vle-vse/data/layer.h +++ /dev/null @@ -1,12 +0,0 @@ - -#include - -typedef struct { - uint32_t M; - uint32_t N; - uint32_t K; - uint32_t Mp; - uint32_t Np; - uint32_t Kp; -} vme_gemm_layer; - diff --git a/sw/vme/apps/tew32/fp32-gemm-vle-vse/kernel/gemm.c b/sw/vme/apps/tew32/fp32-gemm-vle-vse/kernel/gemm.c deleted file mode 100644 index 242ec0647..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vle-vse/kernel/gemm.c +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "gemm.h" - -/* - * Keep the hand-scheduled bias-load M64xN64xK64 implementation as the fast - * path without changing its source. It only reads gemm_l.M/N/K to construct - * panel-local pointers, so bind those fields to the fixed panel shape. - */ -static const struct { - uint32_t M; - uint32_t N; - uint32_t K; -} gemm_tuned_shape = { - .M = 64, - .N = 64, - .K = 64, -}; - -#define gemm_l gemm_tuned_shape -#include "../../fp32-matmul-vle-vse/kernel/gemm.c" -#undef gemm_l - -enum { - GEMM_TE = 16, - GEMM_MAX_CORE_M = 32, - GEMM_PANEL_N = 64, - GEMM_PANEL_K = 64, - GEMM_TILE_BYTES = GEMM_TE * GEMM_TE * sizeof(float), - GEMM_TILE_K_BYTES = GEMM_TE * GEMM_PANEL_K * sizeof(float), - GEMM_C_TILE_ROW_BYTES = (GEMM_PANEL_N / GEMM_TE) * GEMM_TILE_BYTES, -}; - -/* - * Runtime edge kernel for one per-core physical M32xN64xK64 panel. - * - * All runtime scheduling is inside this assembly block: - * - N is traversed in one or two 32-column blocks. - * - K / 8 full groups use LMUL=8 grouped loads and eight VTFMMs. - * - K % 8 tail positions use LMUL=1 loads and one VTFMM each. - * - quadrant activation, loops, stores, and block pointer updates use - * scalar branch instructions rather than C control flow. - * - * A/B retain a physical K stride of 64. C uses packed 16x16 tile-major - * layout, matching partial_at() in main.c. - */ -__attribute__((noinline, aligned(64))) void gemm_block_fp32( - void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt) -{ - const uintptr_t rows = (uintptr_t)M; - const uintptr_t depth = (uintptr_t)K; - uintptr_t n_remaining = (uintptr_t)N; - uintptr_t block_count = ((uintptr_t)N + 31u) >> 5; - uintptr_t a0p = (uintptr_t)addrA; - uintptr_t a1p = a0p + GEMM_TILE_K_BYTES; - uintptr_t b0p = (uintptr_t)addrB; - uintptr_t b1p = b0p + GEMM_TILE_K_BYTES; - uintptr_t p0 = (uintptr_t)addrC; - uintptr_t p1 = p0 + GEMM_C_TILE_ROW_BYTES; - uintptr_t q0 = p0; - uintptr_t q1 = p1; - uintptr_t tss0 = GEMM_TSS_MT0_ROW0; - uintptr_t tss4 = GEMM_TSS_MT4_ROW0; - uintptr_t tss8 = GEMM_TSS_MT8_ROW0; - uintptr_t tss12 = GEMM_TSS_MT12_ROW0; - uintptr_t full_groups; - uintptr_t tail_count; - uintptr_t tm; - uintptr_t tn; - uintptr_t scratch; - - (void)alt_fmt; - - asm volatile( - "beqz %[rows], .Lgemm_done_%=\n" - "beqz %[n_remaining], .Lgemm_done_%=\n" - "beqz %[depth], .Lgemm_done_%=\n" - - /* - * M is fixed for this per-core call. A second row tile is active - * exactly when M > 16; otherwise tm carries the true edge size. - */ - "li %[scratch], 16\n" - "mv %[tm], %[scratch]\n" - "bltu %[scratch], %[rows], .Lgemm_tm_ready_%=\n" - "mv %[tm], %[rows]\n" - ".Lgemm_tm_ready_%=:\n" - - /* - * One assembly iteration covers up to two 16-column tiles. - * n_remaining > 16 activates mt4/mt12. - */ - ".Lgemm_n_block_%=:\n" - "li %[scratch], 16\n" - "mv %[tn], %[scratch]\n" - "bltu %[scratch], %[n_remaining], .Lgemm_tn_ready_%=\n" - "mv %[tn], %[n_remaining]\n" - ".Lgemm_tn_ready_%=:\n" - - "msetmtypei 1, 2\n" - "msettm x0, %[tm]\n" - "li %[scratch], 8\n" - "msettk x0, %[scratch]\n" - "msettn x0, %[tn]\n" - "vsetvli %[scratch], x0, e32, m8, ta, ma\n" - - // Load mt0 - "vle32.v v24, (%[q0])\n" "addi %[q0], %[q0], 512\n" - "vtmv.t.v %[tss0], v24\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v25\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v26\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v27\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v28\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v29\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v30\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v31\n" "addi %[tss0], %[tss0], 1\n" - "vle32.v v8, (%[q0])\n" "addi %[q0], %[q0], -512\n" - "vtmv.t.v %[tss0], v8\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v9\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v10\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v11\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v12\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v13\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v14\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v15\n" "addi %[tss0], %[tss0], -15\n" - "bgeu %[tn], %[n_remaining], .Lgemm_zero_m_done_%=\n" - - // Load mt4 - "addi %[q0], %[q0], 1024\n" - "vle32.v v24, (%[q0])\n" "addi %[q0], %[q0], 512\n" - "vtmv.t.v %[tss4], v24\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v25\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v26\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v27\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v28\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v29\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v30\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v31\n" "addi %[tss4], %[tss4], 1\n" - "vle32.v v8, (%[q0])\n" "addi %[q0], %[q0], -512\n" - "vtmv.t.v %[tss4], v8\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v9\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v10\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v11\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v12\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v13\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v14\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v15\n" "addi %[tss4], %[tss4], -15\n" - ".Lgemm_zero_m_done_%=:\n" - "bgeu %[tm], %[rows], .Lgemm_zero_done_%=\n" - - // Load mt8 - "vle32.v v16, (%[q1])\n" "addi %[q1], %[q1], 512\n" - "vtmv.t.v %[tss8], v16\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v17\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v18\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v19\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v20\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v21\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v22\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v23\n" "addi %[tss8], %[tss8], 1\n" - "vle32.v v8, (%[q1])\n" "addi %[q1], %[q1], -512\n" - "vtmv.t.v %[tss8], v8\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v9\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v10\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v11\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v12\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v13\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v14\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v15\n" "addi %[tss8], %[tss8], -15\n" - "bgeu %[tn], %[n_remaining], .Lgemm_zero_done_%=\n" - - // Load mt12 - "addi %[q1], %[q1], 1024\n" - "vle32.v v24, (%[q1])\n" "addi %[q1], %[q1], 512\n" - "vtmv.t.v %[tss12], v24\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v25\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v26\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v27\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v28\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v29\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v30\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v31\n" "addi %[tss12], %[tss12], 1\n" - "vle32.v v8, (%[q1])\n" "addi %[q1], %[q1], -512\n" - "vtmv.t.v %[tss12], v8\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v9\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v10\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v11\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v12\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v13\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v14\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v15\n" "addi %[tss12], %[tss12], -15\n" - ".Lgemm_zero_done_%=:\n" - - /* - * Full software K groups. One m8 VLE fills eight architectural - * registers; the eight VTFMMs consume those registers as eight - * independent outer products. - */ - "srli %[full_groups], %[depth], 3\n" - "andi %[tail_count], %[depth], 7\n" - "beqz %[full_groups], .Lgemm_tail_setup_%=\n" - - ".Lgemm_full_loop_%=:\n" - "vle32.v v0, (%[a0p])\n" - "vle32.v v16, (%[b0p])\n" - - "vtfmm.tvv mt0, v0, v16\n" - "bgeu %[tn], %[n_remaining], .Lgemm_full_b1_loaded_%=\n" - "vle32.v v24, (%[b1p])\n" - ".Lgemm_full_b1_loaded_%=:\n" - "vtfmm.tvv mt0, v1, v17\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtfmm.tvv mt0, v3, v19\n" - - "bgeu %[tm], %[rows], .Lgemm_full_a1_loaded_%=\n" - "vle32.v v8, (%[a1p])\n" - ".Lgemm_full_a1_loaded_%=:\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtfmm.tvv mt0, v7, v23\n" - - "bgeu %[tn], %[n_remaining], .Lgemm_full_mt4_done_%=\n" - "vtfmm.tvv mt4, v0, v24\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtfmm.tvv mt4, v7, v31\n" - ".Lgemm_full_mt4_done_%=:\n" - - "bgeu %[tm], %[rows], .Lgemm_full_a_done_%=\n" - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - - "bgeu %[tn], %[n_remaining], .Lgemm_full_a_done_%=\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - ".Lgemm_full_a_done_%=:\n" - - /* - * Keep four independent scalar instructions after the final source - * use before the next grouped VLE can overwrite v0/v16. - */ - "addi %[a0p], %[a0p], 512\n" - "addi %[b0p], %[b0p], 512\n" - "bgeu %[tn], %[n_remaining], .Lgemm_full_b1_advanced_%=\n" - "addi %[b1p], %[b1p], 512\n" - ".Lgemm_full_b1_advanced_%=:\n" - "bgeu %[tm], %[rows], .Lgemm_full_a1_advanced_%=\n" - "addi %[a1p], %[a1p], 512\n" - ".Lgemm_full_a1_advanced_%=:\n" - "addi %[full_groups], %[full_groups], -1\n" - "bnez %[full_groups], .Lgemm_full_loop_%=\n" - - /* - * K remainder. LMUL=1 makes each VLE produce one source vector and - * each loop iteration contributes exactly one outer product. - */ - ".Lgemm_tail_setup_%=:\n" - "beqz %[tail_count], .Lgemm_store_setup_%=\n" - "vsetivli x0, 16, e32, m1, ta, ma\n" - - ".Lgemm_tail_loop_%=:\n" - "vle32.v v0, (%[a0p])\n" - "vle32.v v16, (%[b0p])\n" - - "vtfmm.tvv mt0, v0, v16\n" - "bgeu %[tn], %[n_remaining], .Lgemm_tail_b1_loaded_%=\n" - "vle32.v v24, (%[b1p])\n" - ".Lgemm_tail_b1_loaded_%=:\n" - - "bgeu %[tm], %[rows], .Lgemm_tail_a1_loaded_%=\n" - "vle32.v v8, (%[a1p])\n" - ".Lgemm_tail_a1_loaded_%=:\n" - - "bgeu %[tn], %[n_remaining], .Lgemm_tail_mt4_done_%=\n" - "vtfmm.tvv mt4, v0, v24\n" - ".Lgemm_tail_mt4_done_%=:\n" - - "bgeu %[tm], %[rows], .Lgemm_tail_a_done_%=\n" - "vtfmm.tvv mt8, v8, v16\n" - "bgeu %[tn], %[n_remaining], .Lgemm_tail_a_done_%=\n" - "vtfmm.tvv mt12, v8, v24\n" - ".Lgemm_tail_a_done_%=:\n" - - "addi %[a0p], %[a0p], 64\n" - "addi %[b0p], %[b0p], 64\n" - "bgeu %[tn], %[n_remaining], .Lgemm_tail_b1_advanced_%=\n" - "addi %[b1p], %[b1p], 64\n" - ".Lgemm_tail_b1_advanced_%=:\n" - "bgeu %[tm], %[rows], .Lgemm_tail_a1_advanced_%=\n" - "addi %[a1p], %[a1p], 64\n" - ".Lgemm_tail_a1_advanced_%=:\n" - "addi %[tail_count], %[tail_count], -1\n" - "bnez %[tail_count], .Lgemm_tail_loop_%=\n" - - /* - * Packed tile store. VTMV collects 16 rows into two eight-register - * groups; each VSE writes one 512-byte half tile. - */ - ".Lgemm_store_setup_%=:\n" - "vsetvli %[scratch], x0, e32, m8, ta, ma\n" - - // mt0 -> p0 - "li %[scratch], 0\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p0])\n" "addi %[p0], %[p0], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p0])\n" "addi %[p0], %[p0], 512\n" - - // mt4 -> p0, when the second column tile is active - "bgeu %[tn], %[n_remaining], .Lgemm_store_top_done_%=\n" - "li %[scratch], 0x20000000\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p0])\n" "addi %[p0], %[p0], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p0])\n" "addi %[p0], %[p0], 512\n" - ".Lgemm_store_top_done_%=:\n" - - // No lower row tiles means neither mt8 nor mt12 is stored. - "bgeu %[tm], %[rows], .Lgemm_store_done_%=\n" - - // mt8 -> p1 - "li %[scratch], 0x40000000\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p1])\n" "addi %[p1], %[p1], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p1])\n" "addi %[p1], %[p1], 512\n" - - // mt12 -> p1, when the second column tile is active - "bgeu %[tn], %[n_remaining], .Lgemm_store_done_%=\n" - "li %[scratch], 0x60000000\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p1])\n" "addi %[p1], %[p1], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p1])\n" "addi %[p1], %[p1], 512\n" - - ".Lgemm_store_done_%=:\n" - "addi %[block_count], %[block_count], -1\n" - "beqz %[block_count], .Lgemm_done_%=\n" - - /* - * Rewind A to K=0 and advance B to the next two column tiles. - * Reaching this path implies the current block had two active N tiles, - * so b0p/b1p and p0/p1 have both advanced symmetrically. - */ - "slli %[scratch], %[depth], 6\n" - "sub %[a0p], %[a0p], %[scratch]\n" - "sub %[b0p], %[b0p], %[scratch]\n" - "sub %[b1p], %[b1p], %[scratch]\n" - "bgeu %[tm], %[rows], .Lgemm_a1_rewound_%=\n" - "sub %[a1p], %[a1p], %[scratch]\n" - ".Lgemm_a1_rewound_%=:\n" - "li %[scratch], 8192\n" - "add %[b0p], %[b0p], %[scratch]\n" - "add %[b1p], %[b1p], %[scratch]\n" - - /* - * q0 ended on mt4, so another tile reaches the next block's mt0. - * q1 ended on mt12 only when the lower row tile was active; otherwise - * it needs both tile advances here. - */ - "addi %[q0], %[q0], 1024\n" - "addi %[q1], %[q1], 1024\n" - "bltu %[tm], %[rows], .Lgemm_q1_advanced_%=\n" - "addi %[q1], %[q1], 1024\n" - ".Lgemm_q1_advanced_%=:\n" - "addi %[n_remaining], %[n_remaining], -32\n" - "j .Lgemm_n_block_%=\n" - - ".Lgemm_done_%=:\n" - : [a0p] "+&r"(a0p), - [a1p] "+&r"(a1p), - [b0p] "+&r"(b0p), - [b1p] "+&r"(b1p), - [p0] "+&r"(p0), - [p1] "+&r"(p1), - [q0] "+&r"(q0), - [q1] "+&r"(q1), - [tss0] "+&r"(tss0), - [tss4] "+&r"(tss4), - [tss8] "+&r"(tss8), - [tss12] "+&r"(tss12), - [n_remaining] "+&r"(n_remaining), - [block_count] "+&r"(block_count), - [full_groups] "=&r"(full_groups), - [tail_count] "=&r"(tail_count), - [tm] "=&r"(tm), - [tn] "=&r"(tn), - [scratch] "=&r"(scratch) - : [rows] "r"(rows), - [depth] "r"(depth) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-gemm-vle-vse/kernel/gemm.h b/sw/vme/apps/tew32/fp32-gemm-vle-vse/kernel/gemm.h deleted file mode 100644 index c854adff8..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vle-vse/kernel/gemm.h +++ /dev/null @@ -1,18 +0,0 @@ - -#include - -static inline uint32_t gemm_cycle(void) { - uint32_t value; - asm volatile("csrr %0, mcycle" : "=r"(value)); - return value; -} - -// Tuned two-core bias GEMM for one packed M64xN64xK64 panel. C must contain -// the packed initial accumulator values before entry. -void gemm_fp32(float *C, const float *Apack, const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi); - -// Runtime edge kernel. Each call handles at most one M32xN64xK64 per-core -// panel; A/B/C retain the physical M64xN64xK64 packed-panel strides. -void gemm_block_fp32(void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt); diff --git a/sw/vme/apps/tew32/fp32-gemm-vle-vse/main.c b/sw/vme/apps/tew32/fp32-gemm-vle-vse/main.c deleted file mode 100644 index ce75ac4b7..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vle-vse/main.c +++ /dev/null @@ -1,472 +0,0 @@ -#include -#include - -#include DATAHEADER -#ifdef GEMM_KERNEL_SOURCE -#include GEMM_KERNEL_SOURCE -#else -#include KERNELHEADER -#endif - -enum { - TCDM_BYTES = 256 * 1024, - M_CHUNK = 64, - N_CHUNK = 64, - K_CHUNK = 64, - TILE_DIM = 16, - TILE_FLOATS = TILE_DIM * K_CHUNK, - TILE_BYTES = TILE_FLOATS * sizeof(float), - A_TILES = M_CHUNK / TILE_DIM, - B_TILES = N_CHUNK / TILE_DIM, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_TILE_BYTES = TILE_DIM * TILE_DIM * sizeof(float), - C_TILE_ROW_BYTES = B_TILES * C_TILE_BYTES, - C_BYTES = M_CHUNK * N_CHUNK * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - PARTIAL_FLOATS = M_CHUNK * N_CHUNK, - BUFFER_BYTES = C_OFFSET + C_BYTES, - BUFFER_STRIDE = TCDM_BYTES / 2, - BUFFER_COUNT = 2, - L1_BYTES = BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(BUFFER_BYTES <= BUFFER_STRIDE, - "one M64xN64xK64 buffer must fit in half of TCDM"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "ping-pong GEMM buffers exceed TCDM"); -_Static_assert(C_TILE_ROW_BYTES * A_TILES == C_BYTES, - "packed C tile geometry must cover one output panel"); - -static const float zero_tile_dram[TILE_FLOATS] - __attribute__((section(".dram"), aligned(64))) = {0}; - -static uint8_t *l1_base; -static float *partials; -static float *c_out; -static uint32_t allocation_failed; -static volatile uint32_t core_cycles[2]; -static volatile uint32_t verify_errors[2]; -static volatile float verify_max_abs[2]; - -static inline uint32_t min_u32(uint32_t a, uint32_t b) { - return a < b ? a : b; -} - -static inline uint32_t float_bits(float value) { - union { - float f; - uint32_t u; - } bits = {.f = value}; - return bits.u; -} - -static inline void gemm_barrier(uint32_t active_cores) { - if (active_cores > 1) - snrt_cluster_hw_barrier(); -} - -static inline uint32_t chunk_extent(uint32_t total, uint32_t block, - uint32_t chunk) { - const uint32_t offset = block * chunk; - return offset < total ? min_u32(chunk, total - offset) : 0; -} - -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { - // A 1D copy across tiles is legal only when the source tile has the same - // physical K stride as the local panel. - if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) - return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - - const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( - dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); - if (valid_tiles == 0 || valid_k == 0) - return clear_tid; - - const size_t copy_bytes = valid_k * TILE_DIM * sizeof(float); - const size_t src_stride = src_tile_k * TILE_DIM * sizeof(float); - return snrt_dma_start_2d(dst, src, copy_bytes, TILE_BYTES, src_stride, - valid_tiles); -} - -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t k0 = kb * K_CHUNK; - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - const uint32_t mt_count = (gemm_l.M + TILE_DIM - 1) / TILE_DIM; - const uint32_t first_mt = mb * A_TILES; - const uint32_t valid_tiles = first_mt < mt_count - ? min_u32(A_TILES, mt_count - first_mt) - : 0; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : gemm_Apack_dram + - (first_mt * gemm_l.Kp + k0) * TILE_DIM; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, gemm_l.Kp); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t k0 = kb * K_CHUNK; - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - const uint32_t nt_count = (gemm_l.N + TILE_DIM - 1) / TILE_DIM; - const uint32_t first_nt = nb * B_TILES; - const uint32_t valid_tiles = first_nt < nt_count - ? min_u32(B_TILES, nt_count - first_nt) - : 0; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : gemm_Bpack_dram + - (first_nt * gemm_l.Kp + k0) * TILE_DIM; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, gemm_l.Kp); -} - -static snrt_dma_txid_t fill_c(float *dst, uint32_t mb, uint32_t nb, - uint32_t kb) { - // Every K panel produces an independent packed partial. Only kb=0 starts - // from the bias; later panels must start from zero before L3 reduction. - const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( - dst, zero_tile_dram, C_TILE_BYTES, C_TILE_BYTES, 0, - A_TILES * B_TILES); - if (kb != 0) - return clear_tid; - - const uint32_t mt_count = (gemm_l.M + TILE_DIM - 1) / TILE_DIM; - const uint32_t nt_count = (gemm_l.N + TILE_DIM - 1) / TILE_DIM; - const uint32_t first_mt = mb * A_TILES; - const uint32_t first_nt = nb * B_TILES; - const uint32_t valid_mt = first_mt < mt_count - ? min_u32(A_TILES, mt_count - first_mt) - : 0; - const uint32_t valid_nt = first_nt < nt_count - ? min_u32(B_TILES, nt_count - first_nt) - : 0; - if (valid_mt == 0 || valid_nt == 0) - return clear_tid; - - const float *src = gemm_Biaspack_dram + - (first_mt * nt_count + first_nt) * - TILE_DIM * TILE_DIM; - const size_t copy_bytes = valid_nt * C_TILE_BYTES; - const size_t src_stride = nt_count * C_TILE_BYTES; - return snrt_dma_start_2d(dst, src, copy_bytes, - C_TILE_ROW_BYTES, src_stride, valid_mt); -} - -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - fill_b((float *)(base + B_OFFSET), nb, kb); - return fill_c((float *)(base + C_OFFSET), mb, nb, kb); -} - -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - const uint32_t tile = (row / TILE_DIM) * B_TILES + col / TILE_DIM; - return partial[tile * TILE_DIM * TILE_DIM + - (row % TILE_DIM) * TILE_DIM + col % TILE_DIM]; -} - -static float reference_partial(uint32_t row, uint32_t col, uint32_t kb) { - const uint32_t k_begin = kb * K_CHUNK; - const uint32_t k_end = min_u32(k_begin + K_CHUNK, gemm_l.K); - const uint32_t mt = row / TILE_DIM; - const uint32_t nt = col / TILE_DIM; - const uint32_t row_in_tile = row % TILE_DIM; - const uint32_t col_in_tile = col % TILE_DIM; - float sum = 0.0f; - - if (kb == 0) { - const int32_t value = - (int32_t)((row * 3u + col * 5u) & 15u) - 8; - sum = (float)value * 0.03125f; - } - - for (uint32_t k = k_begin; k < k_end; ++k) { - const float a = gemm_Apack_dram[ - (mt * gemm_l.Kp + k) * TILE_DIM + row_in_tile]; - const float b = gemm_Bpack_dram[ - (nt * gemm_l.Kp + k) * TILE_DIM + col_in_tile]; - sum += a * b; - } - return sum; -} - -static inline void finish_spatz(void) { - uint32_t fcsr; - asm volatile("vtdiscard\n" - "csrr %0, fcsr\n" - "fence rw, rw\n" - : "=&r"(fcsr) - : - : "memory"); -} - -int main(void) { - const uint32_t cid = snrt_cluster_core_idx(); - const uint32_t ncores = snrt_cluster_core_num(); -#ifdef SPATZ_GEMM_SINGLE_CORE - const uint32_t active_cores = 1; -#else - const uint32_t active_cores = - gemm_l.M <= 2 * TILE_DIM ? 1 : min_u32(ncores, 2); -#endif - const uint32_t post_cores = min_u32(ncores, 2); - const uint32_t row_pairs = M_CHUNK / (2 * TILE_DIM); - const uint32_t mb_count = (gemm_l.M + M_CHUNK - 1) / M_CHUNK; - const uint32_t nb_count = (gemm_l.N + N_CHUNK - 1) / N_CHUNK; - const uint32_t kb_count = (gemm_l.K + K_CHUNK - 1) / K_CHUNK; - const uint32_t jobs = mb_count * nb_count; - const uint32_t iterations = jobs * kb_count; - - if (cid == 0) { - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc(iterations * PARTIAL_FLOATS * - sizeof(float)); - c_out = (float *)snrt_l3alloc(gemm_l.Mp * gemm_l.Np * sizeof(float)); - allocation_failed = l1_base == 0 || partials == 0 || c_out == 0 || - ncores < active_cores; - } - snrt_cluster_hw_barrier(); - if (allocation_failed) { - if (cid == 0) - printf("general GEMM allocation/core-count failure\n"); - return 2; - } - - snrt_dma_txid_t input_tid[BUFFER_COUNT] = {0, 0}; - uint32_t current_buffer = 0; - uint32_t kernel_cycles = 0; - - if (cid == 0) { - input_tid[0] = fill_buffer(0, 0, 0, 0); - snrt_dma_wait(input_tid[0]); - } - gemm_barrier(active_cores); - - // Prime the alternate buffer. Iteration 0 computes from buffer 0 while - // this descriptor chain fills buffer 1. - if (cid == 0 && iterations > 1) { - const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); - } - - for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(gemm_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(gemm_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - - const uint32_t next_iter = iter + 1; - const uint32_t has_next = next_iter < iterations; - const uint32_t refill_iter = iter + 2; - const uint32_t has_refill = refill_iter < iterations; - const uint32_t next_buffer = current_buffer ^ 1; - - if (cid < active_cores) { - uint8_t *base = l1_base + current_buffer * BUFFER_STRIDE; - float *a = (float *)(base + A_OFFSET); - float *b = (float *)(base + B_OFFSET); - float *c = (float *)(base + C_OFFSET); - const uint32_t t0 = gemm_cycle(); - uint32_t did_compute = 0; - - if (valid_m == M_CHUNK && valid_n == N_CHUNK && - valid_k == K_CHUNK) { - for (uint32_t pair = cid; pair < row_pairs; - pair += active_cores) { - const uint32_t ti_lo = 2 * pair; - gemm_fp32(c, a, b, ti_lo, ti_lo + 2); - did_compute = 1; - } - } else { - for (uint32_t pair = cid; pair < row_pairs; - pair += active_cores) { - const uint32_t core_row = pair * 2 * TILE_DIM; - const uint32_t core_m = core_row < valid_m - ? min_u32(2 * TILE_DIM, - valid_m - core_row) - : 0; - if (core_m != 0) { - gemm_block_fp32( - (void *)((uint8_t *)a + - core_row * K_CHUNK * sizeof(float)), - (void *)b, - (void *)((uint8_t *)c + - core_row * N_CHUNK * sizeof(float)), - (int)valid_k, (int)valid_n, (int)core_m, 0); - did_compute = 1; - } - } - } - - if (did_compute) { - finish_spatz(); - kernel_cycles += gemm_cycle() - t0; - } - } - gemm_barrier(active_cores); - - if (cid == 0) { - // Do not switch to the alternate buffer until all of its A/B - // descriptors have completed. - if (has_next) - snrt_dma_wait(input_tid[next_buffer]); - - float *src = (float *)(l1_base + - current_buffer * BUFFER_STRIDE + C_OFFSET); - snrt_dma_start_1d(partials + iter * PARTIAL_FLOATS, src, C_BYTES); - - // Reuse the just-computed buffer for iteration i+2. The DMA - // executes descriptors in issue order, so C(i) is read before the - // following A/B refill overwrites this 128 KiB half. This complete - // chain overlaps compute(i+1) in the other half. - if (has_refill) { - const uint32_t refill_job = refill_iter / kb_count; - input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, - refill_job % nb_count, refill_iter % kb_count); - } - } - gemm_barrier(active_cores); - if (has_next) - current_buffer = next_buffer; - } - - if (cid < active_cores) - core_cycles[cid] = kernel_cycles; - - if (cid == 0) - snrt_dma_wait_all(); - snrt_cluster_hw_barrier(); - - if (cid < post_cores) { - const uint32_t output_elements = gemm_l.M * gemm_l.N; - for (uint32_t idx = cid; idx < output_elements; idx += post_cores) { - const uint32_t row = idx / gemm_l.N; - const uint32_t col = idx % gemm_l.N; - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - float sum = 0.0f; - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float *partial = partials + - (job * kb_count + kb) * PARTIAL_FLOATS; - sum += partial_at(partial, local_row, local_col); - } - c_out[row * gemm_l.Np + col] = sum; - } - } - snrt_cluster_hw_barrier(); - - uint32_t cycles = core_cycles[0]; - for (uint32_t core = 1; core < active_cores; ++core) { - if (core_cycles[core] > cycles) - cycles = core_cycles[core]; - } - - enum { CE = 8 }; - const uint64_t useful_fmas = - (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE * active_cores; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - uint32_t local_errors = 0; - uint32_t printed_partial_debug = 0; - float local_max_abs = 0.0f; - const float tolerance = 1.0e-3f; - for (uint32_t row = cid; cid < post_cores && row < gemm_l.M; - row += post_cores) { - uint32_t row_errors = 0; - uint32_t error_mask_lo = 0; - uint32_t error_mask_hi = 0; - for (uint32_t col = 0; col < gemm_l.N; ++col) { - const uint32_t idx = row * gemm_l.N + col; - const float output = c_out[row * gemm_l.Np + col]; - const float expected = gemm_Cref_dram[idx]; - if (float_bits(output) == float_bits(expected)) - continue; - const float diff = output - expected; - const float abs = diff < 0.0f ? -diff : diff; - float ref = expected; - ref = ref < 0.0f ? -ref : ref; - if (abs > local_max_abs) - local_max_abs = abs; - if (abs > tolerance && abs > tolerance * ref) { - ++local_errors; - ++row_errors; - if (col < 32) - error_mask_lo |= 1u << col; - else if (col < 64) - error_mask_hi |= 1u << (col - 32); - - if (!printed_partial_debug) { - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - printf("first mismatch row=%u col=%u output=0x%08x ref=0x%08x\n", - row, col, *(uint32_t *)&c_out[row * gemm_l.Np + col], - *(uint32_t *)&gemm_Cref_dram[idx]); - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float actual = partial_at( - partials + (job * kb_count + kb) * PARTIAL_FLOATS, - local_row, local_col); - const float expected = reference_partial(row, col, kb); - printf(" kb=%u actual=0x%08x expected=0x%08x\n", kb, - *(uint32_t *)&actual, *(uint32_t *)&expected); - } - printed_partial_debug = 1; - } - } - } - if (row_errors) - printf("core %u row %u: %u errors, col_mask[63:0]=%08x_%08x\n", - cid, - row, row_errors, error_mask_hi, error_mask_lo); - } - - if (cid < post_cores) { - verify_errors[cid] = local_errors; - verify_max_abs[cid] = local_max_abs; - } - snrt_cluster_hw_barrier(); - - if (cid != 0) - return 0; - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t core = 0; core < post_cores; ++core) { - errors += verify_errors[core]; - if (verify_max_abs[core] > max_abs) - max_abs = verify_max_abs[core]; - } - - printf("\n=== General FP32 GEMM %ux%ux%u " - "(M64xN64xK64 panels, 2 buffers, %u core%s) ===\n", - gemm_l.M, gemm_l.N, gemm_l.K, active_cores, - active_cores == 1 ? "" : "s"); - printf("Cycles: %u\n", cycles); - printf("Utilization: %u.%02u%%\n", utilization_bp / 100, - utilization_bp % 100); - printf("MaxAbsErr: 0x%08x\n", *(uint32_t *)&max_abs); - printf("Errors: %u\n", errors); - printf("Status: %s\n", errors ? "FAIL" : "PASS"); - return errors ? 1 : 0; -} diff --git a/sw/vme/apps/tew32/fp32-gemm-vle-vse/script/gen_data.py b/sw/vme/apps/tew32/fp32-gemm-vle-vse/script/gen_data.py deleted file mode 100644 index 07d0ce603..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vle-vse/script/gen_data.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib - -import numpy as np - -def align_up(value, alignment): - return (value + alignment - 1) // alignment * alignment - -def c_array(array): - return "{" + ", ".join(repr(float(x)) + "f" for x in array.flat) + "}" - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--m", type=int, required=True) - parser.add_argument("--n", type=int, required=True) - parser.add_argument("--k", type=int, required=True) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output", type=pathlib.Path, required=True) - args = parser.parse_args() - - if min(args.m, args.n, args.k) <= 0: - raise ValueError("M, N, and K must all be positive") - - mp = align_up(args.m, 16) - npad = align_up(args.n, 16) - kp = align_up(args.k, 8) - rng = np.random.default_rng(args.seed) - a = rng.standard_normal((args.m, args.k), dtype=np.float32) - b = rng.standard_normal((args.k, args.n), dtype=np.float32) - - row = np.arange(args.m, dtype=np.int64)[:, None] - col = np.arange(args.n, dtype=np.int64)[None, :] - bias = (((row * 3 + col * 5) & 15) - 8).astype(np.float32) * 0.03125 - c = (a @ b).astype(np.float32) + bias - - ap = np.zeros((mp, kp), dtype=np.float32) - bp = np.zeros((kp, npad), dtype=np.float32) - biasp = np.zeros((mp, npad), dtype=np.float32) - ap[:args.m, :args.k] = a - bp[:args.k, :args.n] = b - biasp[:args.m, :args.n] = bias - - apack = ap.reshape(mp // 16, 16, kp // 8, 8).transpose(0, 2, 3, 1).copy() - bpack = bp.reshape(kp // 8, 8, npad // 16, 16).transpose(2, 0, 1, 3).copy() - biaspack = biasp.reshape(mp // 16, 16, npad // 16, 16).transpose( - 0, 2, 1, 3).copy() - - text = f'''// Auto-generated by fp32-gemm-vle-vse/script/gen_data.py. -#include "layer.h" - -static const vme_gemm_layer gemm_l = {{ - .M = {args.m}, .N = {args.n}, .K = {args.k}, - .Mp = {mp}, .Np = {npad}, .Kp = {kp}, -}}; - -static const float gemm_Cref_dram[{args.m * args.n}] - __attribute__((section(".dram"), aligned(64))) = {c_array(c)}; - -static const float gemm_Biaspack_dram[{mp * npad}] - __attribute__((section(".dram"), aligned(64))) = {c_array(biaspack)}; - -static const float gemm_Apack_dram[{mp * kp}] - __attribute__((section(".dram"), aligned(64))) = {c_array(apack)}; - -static const float gemm_Bpack_dram[{kp * npad}] - __attribute__((section(".dram"), aligned(64))) = {c_array(bpack)}; -''' - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(text) - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-gemm-vse/CMakeLists.txt b/sw/vme/apps/tew32/fp32-gemm-vse/CMakeLists.txt deleted file mode 100644 index 1f1193c6c..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vse/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ - -set(VME_GEMM_SHAPES - "64x64x64;64x64x32" CACHE STRING - "Semicolon-separated MxNxK shapes for the general FP32 GEMM tests") - -set(_vme_gemm_build_shapes ${VME_GEMM_SHAPES}) -list(APPEND _vme_gemm_build_shapes - "64x64x57" "64x64x37" - "64x53x64" "53x64x64" "53x53x57" "64x53x57" "53x64x57" - "16x16x64" "15x15x64" "16x32x64" "32x16x64" "15x31x64" - "31x16x64" "15x31x57" "16x48x64" "15x48x64" - "15x47x64" "47x15x64" "47x15x57") -set(_vme_gemm_normalized_shapes) -foreach(_shape IN LISTS _vme_gemm_build_shapes) - string(STRIP "${_shape}" _shape) - list(APPEND _vme_gemm_normalized_shapes "${_shape}") -endforeach() -set(_vme_gemm_build_shapes ${_vme_gemm_normalized_shapes}) -list(REMOVE_DUPLICATES _vme_gemm_build_shapes) - -foreach(_shape IN LISTS _vme_gemm_build_shapes) - if(NOT _shape MATCHES "^([0-9]+)x([0-9]+)x([0-9]+)$") - message(FATAL_ERROR "Invalid GEMM shape '${_shape}', expected MxNxK") - endif() - set(_m ${CMAKE_MATCH_1}) - set(_n ${CMAKE_MATCH_2}) - set(_k ${CMAKE_MATCH_3}) - set(_target fp32-gemm-vse_M${_m}_N${_n}_K${_k}) - set(_generated_dir ${CMAKE_CURRENT_BINARY_DIR}/generated) - set(_header ${_generated_dir}/data/data_${_m}_${_n}_${_k}.h) - - add_custom_command( - OUTPUT ${_header} - COMMAND ${CMAKE_COMMAND} -E make_directory ${_generated_dir}/data - COMMAND ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py - --m ${_m} --n ${_n} --k ${_k} --output ${_header} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py - ${CMAKE_CURRENT_SOURCE_DIR}/../fp32-gemm-vtse/script/gen_data.py - ${CMAKE_CURRENT_SOURCE_DIR}/data/layer.h - VERBATIM) - - if(BUILD_TESTS) - add_custom_target(generate-${_target} DEPENDS ${_header}) - add_snitch_test(${_target} main.c) - add_dependencies(test-${SNITCH_TEST_PREFIX}${_target} generate-${_target}) - target_include_directories(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/data - ${_generated_dir}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - DATAHEADER="data/data_${_m}_${_n}_${_k}.h" - KERNELHEADER="kernel/gemm.c") - endif() -endforeach() diff --git a/sw/vme/apps/tew32/fp32-gemm-vse/data/layer.h b/sw/vme/apps/tew32/fp32-gemm-vse/data/layer.h deleted file mode 100644 index 76e859a50..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vse/data/layer.h +++ /dev/null @@ -1,12 +0,0 @@ - -#include - -typedef struct { - uint32_t M; - uint32_t N; - uint32_t K; - uint32_t Mp; - uint32_t Np; - uint32_t Kp; -} vme_gemm_layer; - diff --git a/sw/vme/apps/tew32/fp32-gemm-vse/kernel/gemm.c b/sw/vme/apps/tew32/fp32-gemm-vse/kernel/gemm.c deleted file mode 100644 index 0bcb100dd..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vse/kernel/gemm.c +++ /dev/null @@ -1,324 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "gemm.h" - -enum { - GEMM_TE = 16, - GEMM_MAX_CORE_M = 32, - GEMM_PANEL_N = 64, - GEMM_PANEL_K = 64, - GEMM_TILE_BYTES = GEMM_TE * GEMM_TE * sizeof(float), - GEMM_TILE_K_BYTES = GEMM_TE * GEMM_PANEL_K * sizeof(float), - GEMM_C_TILE_ROW_BYTES = (GEMM_PANEL_N / GEMM_TE) * GEMM_TILE_BYTES, -}; - -/* - * Runtime edge kernel for one per-core physical M32xN64xK64 panel. - * - * All runtime scheduling is inside this assembly block: - * - N is traversed in one or two 32-column blocks. - * - K / 4 full groups use LMUL=4 grouped loads and one fused VTFMM. - * - K % 4 tail positions use LMUL=1 loads and one VTFMM each. - * - quadrant activation, loops, stores, and block pointer updates use - * scalar branch instructions rather than C control flow. - * - * A/B retain a physical K stride of 64. C uses packed 16x16 tile-major - * layout, matching partial_at() in main.c. - */ -__attribute__((noinline, aligned(64))) void gemm_fp32( - void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt) -{ - const uintptr_t rows = (uintptr_t)M; - const uintptr_t depth = (uintptr_t)K; - uintptr_t n_remaining = (uintptr_t)N; - uintptr_t block_count = ((uintptr_t)N + 31u) >> 5; - uintptr_t a0p = (uintptr_t)addrA; - uintptr_t a1p = a0p + GEMM_TILE_K_BYTES; - uintptr_t b0p = (uintptr_t)addrB; - uintptr_t b1p = b0p + GEMM_TILE_K_BYTES; - uintptr_t p0 = (uintptr_t)addrC; - uintptr_t p1 = p0 + GEMM_C_TILE_ROW_BYTES; - uintptr_t full_groups; - uintptr_t tail_count; - uintptr_t tm; - uintptr_t tn; - uintptr_t scratch; - - (void)alt_fmt; - - asm volatile( - "beqz %[rows], .Lgemm_done_%=\n" - "beqz %[n_remaining], .Lgemm_done_%=\n" - "beqz %[depth], .Lgemm_done_%=\n" - - /* - * M is fixed for this per-core call. A second row tile is active - * exactly when M > 16; otherwise tm carries the true edge size. - */ - "li %[scratch], 16\n" - "mv %[tm], %[scratch]\n" - "bltu %[scratch], %[rows], .Lgemm_tm_ready_%=\n" - "mv %[tm], %[rows]\n" - ".Lgemm_tm_ready_%=:\n" - - /* - * One assembly iteration covers up to two 16-column tiles. - * n_remaining > 16 activates mt4/mt12. - */ - ".Lgemm_n_block_%=:\n" - "li %[scratch], 16\n" - "mv %[tn], %[scratch]\n" - "bltu %[scratch], %[n_remaining], .Lgemm_tn_ready_%=\n" - "mv %[tn], %[n_remaining]\n" - ".Lgemm_tn_ready_%=:\n" - - "msetmtypei 1, 2\n" - "msettm x0, %[tm]\n" - "li %[scratch], 4\n" - "msettk x0, %[scratch]\n" - "msettn x0, %[tn]\n" - - "vtzero mt0\n" - "bgeu %[tn], %[n_remaining], .Lgemm_zero_m_done_%=\n" - "vtzero mt4\n" - ".Lgemm_zero_m_done_%=:\n" - "bgeu %[tm], %[rows], .Lgemm_zero_done_%=\n" - "vtzero mt8\n" - "bgeu %[tn], %[n_remaining], .Lgemm_zero_done_%=\n" - "vtzero mt12\n" - ".Lgemm_zero_done_%=:\n" - - /* - * Full software K groups. One m4 VLE fills four architectural - * registers and one VTFMM consumes them as mtype.tk=4 K rows. - */ - "srli %[full_groups], %[depth], 2\n" - "andi %[tail_count], %[depth], 3\n" - "beqz %[full_groups], .Lgemm_tail_setup_%=\n" - "vsetvli %[scratch], x0, e32, m4, ta, ma\n" - - ".Lgemm_full_loop_%=:\n" - "vle32.v v0, (%[a0p])\n" - "vle32.v v16, (%[b0p])\n" - "bgeu %[tn], %[n_remaining], .Lgemm_full_b1_loaded_%=\n" - "vle32.v v24, (%[b1p])\n" - ".Lgemm_full_b1_loaded_%=:\n" - - "vtfmm.tvv mt0, v0, v16\n" - - "bgeu %[tm], %[rows], .Lgemm_full_a1_loaded_%=\n" - "vle32.v v8, (%[a1p])\n" - ".Lgemm_full_a1_loaded_%=:\n" - - "bgeu %[tn], %[n_remaining], .Lgemm_full_mt4_done_%=\n" - "vtfmm.tvv mt4, v0, v24\n" - ".Lgemm_full_mt4_done_%=:\n" - - "bgeu %[tm], %[rows], .Lgemm_full_a_done_%=\n" - "vtfmm.tvv mt8, v8, v16\n" - - "bgeu %[tn], %[n_remaining], .Lgemm_full_a_done_%=\n" - "vtfmm.tvv mt12, v8, v24\n" - ".Lgemm_full_a_done_%=:\n" - - /* - * Keep four independent scalar instructions after the final source - * use before the next grouped VLE can overwrite v0/v16. - */ - "addi %[a0p], %[a0p], 256\n" - "addi %[b0p], %[b0p], 256\n" - "bgeu %[tn], %[n_remaining], .Lgemm_full_b1_advanced_%=\n" - "addi %[b1p], %[b1p], 256\n" - ".Lgemm_full_b1_advanced_%=:\n" - "bgeu %[tm], %[rows], .Lgemm_full_a1_advanced_%=\n" - "addi %[a1p], %[a1p], 256\n" - ".Lgemm_full_a1_advanced_%=:\n" - "addi %[full_groups], %[full_groups], -1\n" - "bnez %[full_groups], .Lgemm_full_loop_%=\n" - - /* - * K remainder. LMUL=1 makes each VLE produce one source vector and - * each loop iteration contributes exactly one outer product. - */ - ".Lgemm_tail_setup_%=:\n" - "beqz %[tail_count], .Lgemm_store_setup_%=\n" - "li %[scratch], 1\n" - "msettk x0, %[scratch]\n" - "vsetivli x0, 16, e32, m1, ta, ma\n" - - ".Lgemm_tail_loop_%=:\n" - "vle32.v v0, (%[a0p])\n" - "vle32.v v16, (%[b0p])\n" - "bgeu %[tn], %[n_remaining], .Lgemm_tail_b1_loaded_%=\n" - "vle32.v v24, (%[b1p])\n" - ".Lgemm_tail_b1_loaded_%=:\n" - - "vtfmm.tvv mt0, v0, v16\n" - - "bgeu %[tm], %[rows], .Lgemm_tail_a1_loaded_%=\n" - "vle32.v v8, (%[a1p])\n" - ".Lgemm_tail_a1_loaded_%=:\n" - - "bgeu %[tn], %[n_remaining], .Lgemm_tail_mt4_done_%=\n" - "vtfmm.tvv mt4, v0, v24\n" - ".Lgemm_tail_mt4_done_%=:\n" - - "bgeu %[tm], %[rows], .Lgemm_tail_a_done_%=\n" - "vtfmm.tvv mt8, v8, v16\n" - "bgeu %[tn], %[n_remaining], .Lgemm_tail_a_done_%=\n" - "vtfmm.tvv mt12, v8, v24\n" - ".Lgemm_tail_a_done_%=:\n" - - "addi %[a0p], %[a0p], 64\n" - "addi %[b0p], %[b0p], 64\n" - "bgeu %[tn], %[n_remaining], .Lgemm_tail_b1_advanced_%=\n" - "addi %[b1p], %[b1p], 64\n" - ".Lgemm_tail_b1_advanced_%=:\n" - "bgeu %[tm], %[rows], .Lgemm_tail_a1_advanced_%=\n" - "addi %[a1p], %[a1p], 64\n" - ".Lgemm_tail_a1_advanced_%=:\n" - "addi %[tail_count], %[tail_count], -1\n" - "bnez %[tail_count], .Lgemm_tail_loop_%=\n" - - /* - * Packed tile store. VTMV collects 16 rows into two eight-register - * groups; each VSE writes one 512-byte half tile. - */ - ".Lgemm_store_setup_%=:\n" - "vsetvli %[scratch], x0, e32, m8, ta, ma\n" - - // mt0 -> p0 - "li %[scratch], 0\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p0])\n" "addi %[p0], %[p0], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p0])\n" "addi %[p0], %[p0], 512\n" - - // mt4 -> p0, when the second column tile is active - "bgeu %[tn], %[n_remaining], .Lgemm_store_top_done_%=\n" - "li %[scratch], 0x20000000\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p0])\n" "addi %[p0], %[p0], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p0])\n" "addi %[p0], %[p0], 512\n" - ".Lgemm_store_top_done_%=:\n" - - // No lower row tiles means neither mt8 nor mt12 is stored. - "bgeu %[tm], %[rows], .Lgemm_store_done_%=\n" - - // mt8 -> p1 - "li %[scratch], 0x40000000\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p1])\n" "addi %[p1], %[p1], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p1])\n" "addi %[p1], %[p1], 512\n" - - // mt12 -> p1, when the second column tile is active - "bgeu %[tn], %[n_remaining], .Lgemm_store_done_%=\n" - "li %[scratch], 0x60000000\n" - "vtmv.v.t v0, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v1, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v2, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v3, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v4, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v5, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v6, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v7, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v0, (%[p1])\n" "addi %[p1], %[p1], 512\n" - "vtmv.v.t v8, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v9, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v10, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v11, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v12, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v13, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v14, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vtmv.v.t v15, %[scratch]\n" "addi %[scratch], %[scratch], 1\n" - "vse32.v v8, (%[p1])\n" "addi %[p1], %[p1], 512\n" - - ".Lgemm_store_done_%=:\n" - "addi %[block_count], %[block_count], -1\n" - "beqz %[block_count], .Lgemm_done_%=\n" - - /* - * Rewind A to K=0 and advance B to the next two column tiles. - * Reaching this path implies the current block had two active N tiles, - * so b0p/b1p and p0/p1 have both advanced symmetrically. - */ - "slli %[scratch], %[depth], 6\n" - "sub %[a0p], %[a0p], %[scratch]\n" - "sub %[b0p], %[b0p], %[scratch]\n" - "sub %[b1p], %[b1p], %[scratch]\n" - "bgeu %[tm], %[rows], .Lgemm_a1_rewound_%=\n" - "sub %[a1p], %[a1p], %[scratch]\n" - ".Lgemm_a1_rewound_%=:\n" - "li %[scratch], 8192\n" - "add %[b0p], %[b0p], %[scratch]\n" - "add %[b1p], %[b1p], %[scratch]\n" - "addi %[n_remaining], %[n_remaining], -32\n" - "j .Lgemm_n_block_%=\n" - - ".Lgemm_done_%=:\n" - : [a0p] "+&r"(a0p), - [a1p] "+&r"(a1p), - [b0p] "+&r"(b0p), - [b1p] "+&r"(b1p), - [p0] "+&r"(p0), - [p1] "+&r"(p1), - [n_remaining] "+&r"(n_remaining), - [block_count] "+&r"(block_count), - [full_groups] "=&r"(full_groups), - [tail_count] "=&r"(tail_count), - [tm] "=&r"(tm), - [tn] "=&r"(tn), - [scratch] "=&r"(scratch) - : [rows] "r"(rows), - [depth] "r"(depth) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-gemm-vse/kernel/gemm.h b/sw/vme/apps/tew32/fp32-gemm-vse/kernel/gemm.h deleted file mode 100644 index 30e0f95ac..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vse/kernel/gemm.h +++ /dev/null @@ -1,13 +0,0 @@ - -#include - -static inline uint32_t gemm_cycle(void) { - uint32_t value; - asm volatile("csrr %0, mcycle" : "=r"(value)); - return value; -} - -// Runtime kernel. Each call handles at most one physical M32xN64xK64 panel; -// A/B/C retain the physical 64x64x64 packed-panel strides. -void gemm_fp32(void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt); diff --git a/sw/vme/apps/tew32/fp32-gemm-vse/main.c b/sw/vme/apps/tew32/fp32-gemm-vse/main.c deleted file mode 100644 index 154de23b2..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vse/main.c +++ /dev/null @@ -1,375 +0,0 @@ -#include -#include - -#include DATAHEADER -#ifdef GEMM_KERNEL_SOURCE -#include GEMM_KERNEL_SOURCE -#else -#include KERNELHEADER -#endif - -enum { - TCDM_BYTES = 256 * 1024, - M_CHUNK = 64, - N_CHUNK = 64, - K_CHUNK = 64, - TILE_DIM = 16, - TILE_FLOATS = TILE_DIM * K_CHUNK, - TILE_BYTES = TILE_FLOATS * sizeof(float), - A_TILES = M_CHUNK / TILE_DIM, - B_TILES = N_CHUNK / TILE_DIM, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_BYTES = M_CHUNK * N_CHUNK * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - PARTIAL_FLOATS = M_CHUNK * N_CHUNK, - BUFFER_BYTES = C_OFFSET + C_BYTES, - BUFFER_STRIDE = TCDM_BYTES / 2, - BUFFER_COUNT = 2, - L1_BYTES = BUFFER_STRIDE + BUFFER_BYTES, -}; - -_Static_assert(BUFFER_BYTES <= BUFFER_STRIDE, - "one M64xN64xK64 buffer must fit in half of TCDM"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "ping-pong GEMM buffers exceed TCDM"); - -static const float zero_tile_dram[TILE_FLOATS] - __attribute__((section(".dram"), aligned(64))) = {0}; - -static uint8_t *l1_base; -static float *partials; -static float *c_out; -static uint32_t allocation_failed; - -static inline uint32_t min_u32(uint32_t a, uint32_t b) { - return a < b ? a : b; -} - -static inline uint32_t float_bits(float value) { - union { - float f; - uint32_t u; - } bits = {.f = value}; - return bits.u; -} - -static inline uint32_t chunk_extent(uint32_t total, uint32_t block, - uint32_t chunk) { - const uint32_t offset = block * chunk; - return offset < total ? min_u32(chunk, total - offset) : 0; -} - -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { - // A 1D copy across tiles is legal only when the source tile has the same - // physical K stride as the local panel. - if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) - return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - - const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( - dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); - if (valid_tiles == 0 || valid_k == 0) - return clear_tid; - - const size_t copy_bytes = valid_k * TILE_DIM * sizeof(float); - const size_t src_stride = src_tile_k * TILE_DIM * sizeof(float); - return snrt_dma_start_2d(dst, src, copy_bytes, TILE_BYTES, src_stride, - valid_tiles); -} - -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t k0 = kb * K_CHUNK; - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - const uint32_t mt_count = (gemm_l.M + TILE_DIM - 1) / TILE_DIM; - const uint32_t first_mt = mb * A_TILES; - const uint32_t valid_tiles = first_mt < mt_count - ? min_u32(A_TILES, mt_count - first_mt) - : 0; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : gemm_Apack_dram + - (first_mt * gemm_l.Kp + k0) * TILE_DIM; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, gemm_l.Kp); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t k0 = kb * K_CHUNK; - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - const uint32_t nt_count = (gemm_l.N + TILE_DIM - 1) / TILE_DIM; - const uint32_t first_nt = nb * B_TILES; - const uint32_t valid_tiles = first_nt < nt_count - ? min_u32(B_TILES, nt_count - first_nt) - : 0; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : gemm_Bpack_dram + - (first_nt * gemm_l.Kp + k0) * TILE_DIM; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, gemm_l.Kp); -} - -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); -} - -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - const uint32_t tile = (row / TILE_DIM) * B_TILES + col / TILE_DIM; - return partial[tile * TILE_DIM * TILE_DIM + - (row % TILE_DIM) * TILE_DIM + col % TILE_DIM]; -} - -static float reference_partial(uint32_t row, uint32_t col, uint32_t kb) { - const uint32_t k_begin = kb * K_CHUNK; - const uint32_t k_end = min_u32(k_begin + K_CHUNK, gemm_l.K); - const uint32_t mt = row / TILE_DIM; - const uint32_t nt = col / TILE_DIM; - const uint32_t row_in_tile = row % TILE_DIM; - const uint32_t col_in_tile = col % TILE_DIM; - float sum = 0.0f; - - for (uint32_t k = k_begin; k < k_end; ++k) { - const float a = gemm_Apack_dram[ - (mt * gemm_l.Kp + k) * TILE_DIM + row_in_tile]; - const float b = gemm_Bpack_dram[ - (nt * gemm_l.Kp + k) * TILE_DIM + col_in_tile]; - sum += a * b; - } - return sum; -} - -static inline void finish_spatz(void) { - uint32_t fcsr; - asm volatile("vtdiscard\n" - "csrr %0, fcsr\n" - "fence rw, rw\n" - : "=&r"(fcsr) - : - : "memory"); -} - -int main(void) { - const uint32_t cid = snrt_cluster_core_idx(); - const uint32_t row_pairs = M_CHUNK / (2 * TILE_DIM); - const uint32_t mb_count = (gemm_l.M + M_CHUNK - 1) / M_CHUNK; - const uint32_t nb_count = (gemm_l.N + N_CHUNK - 1) / N_CHUNK; - const uint32_t kb_count = (gemm_l.K + K_CHUNK - 1) / K_CHUNK; - const uint32_t jobs = mb_count * nb_count; - const uint32_t iterations = jobs * kb_count; - - if (cid == 0) { - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc(iterations * PARTIAL_FLOATS * - sizeof(float)); - c_out = (float *)snrt_l3alloc(gemm_l.Mp * gemm_l.Np * sizeof(float)); - allocation_failed = l1_base == 0 || partials == 0 || c_out == 0; - } - snrt_cluster_hw_barrier(); - if (allocation_failed) { - if (cid == 0) - printf("general GEMM allocation failure\n"); - return 2; - } - - snrt_dma_txid_t input_tid[BUFFER_COUNT] = {0, 0}; - uint32_t current_buffer = 0; - uint32_t kernel_cycles = 0; - - if (cid == 0) { - input_tid[0] = fill_buffer(0, 0, 0, 0); - snrt_dma_wait(input_tid[0]); - } - snrt_cluster_hw_barrier(); - - // Prime the alternate buffer. Iteration 0 computes from buffer 0 while - // this descriptor chain fills buffer 1. - if (cid == 0 && iterations > 1) { - const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); - } - - for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(gemm_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(gemm_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(gemm_l.K, kb, K_CHUNK); - - const uint32_t next_iter = iter + 1; - const uint32_t has_next = next_iter < iterations; - const uint32_t refill_iter = iter + 2; - const uint32_t has_refill = refill_iter < iterations; - const uint32_t next_buffer = current_buffer ^ 1; - - if (cid == 0) { - uint8_t *base = l1_base + current_buffer * BUFFER_STRIDE; - float *a = (float *)(base + A_OFFSET); - float *b = (float *)(base + B_OFFSET); - float *c = (float *)(base + C_OFFSET); - const uint32_t t0 = gemm_cycle(); - uint32_t did_compute = 0; - - for (uint32_t pair = 0; pair < row_pairs; ++pair) { - const uint32_t panel_row = pair * 2 * TILE_DIM; - const uint32_t panel_m = panel_row < valid_m - ? min_u32(2 * TILE_DIM, - valid_m - panel_row) - : 0; - if (panel_m != 0) { - gemm_fp32( - (void *)((uint8_t *)a + - panel_row * K_CHUNK * sizeof(float)), - (void *)b, - (void *)((uint8_t *)c + - panel_row * N_CHUNK * sizeof(float)), - (int)valid_k, (int)valid_n, (int)panel_m, 0); - did_compute = 1; - } - } - - if (did_compute) { - finish_spatz(); - kernel_cycles += gemm_cycle() - t0; - } - } - snrt_cluster_hw_barrier(); - - if (cid == 0) { - // Do not switch to the alternate buffer until all of its A/B - // descriptors have completed. - if (has_next) - snrt_dma_wait(input_tid[next_buffer]); - - float *src = (float *)(l1_base + - current_buffer * BUFFER_STRIDE + C_OFFSET); - snrt_dma_start_1d(partials + iter * PARTIAL_FLOATS, src, C_BYTES); - - // Reuse the just-computed buffer for iteration i+2. The DMA - // executes descriptors in issue order, so C(i) is read before the - // following A/B refill overwrites this 128 KiB half. This complete - // chain overlaps compute(i+1) in the other half. - if (has_refill) { - const uint32_t refill_job = refill_iter / kb_count; - input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, - refill_job % nb_count, refill_iter % kb_count); - } - } - snrt_cluster_hw_barrier(); - if (has_next) - current_buffer = next_buffer; - } - - if (cid == 0) - snrt_dma_wait_all(); - snrt_cluster_hw_barrier(); - - // The second hart only participates in cluster-wide synchronization. - if (cid != 0) - return 0; - - const uint32_t output_elements = gemm_l.M * gemm_l.N; - for (uint32_t idx = 0; idx < output_elements; ++idx) { - const uint32_t row = idx / gemm_l.N; - const uint32_t col = idx % gemm_l.N; - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - float sum = 0.0f; - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float *partial = partials + - (job * kb_count + kb) * PARTIAL_FLOATS; - sum += partial_at(partial, local_row, local_col); - } - c_out[row * gemm_l.Np + col] = sum; - } - - const uint32_t cycles = kernel_cycles; - - enum { CE = 8 }; - const uint64_t useful_fmas = - (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - uint32_t local_errors = 0; - uint32_t printed_partial_debug = 0; - float local_max_abs = 0.0f; - const float tolerance = 1.0e-3f; - for (uint32_t row = 0; row < gemm_l.M; ++row) { - uint32_t row_errors = 0; - uint32_t error_mask_lo = 0; - uint32_t error_mask_hi = 0; - for (uint32_t col = 0; col < gemm_l.N; ++col) { - const uint32_t idx = row * gemm_l.N + col; - const float output = c_out[row * gemm_l.Np + col]; - const float expected = gemm_Cref_dram[idx]; - if (float_bits(output) == float_bits(expected)) - continue; - const float diff = output - expected; - const float abs = diff < 0.0f ? -diff : diff; - float ref = expected; - ref = ref < 0.0f ? -ref : ref; - if (abs > local_max_abs) - local_max_abs = abs; - if (abs > tolerance && abs > tolerance * ref) { - ++local_errors; - ++row_errors; - if (col < 32) - error_mask_lo |= 1u << col; - else if (col < 64) - error_mask_hi |= 1u << (col - 32); - - if (!printed_partial_debug) { - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - printf("first mismatch row=%u col=%u output=0x%08x ref=0x%08x\n", - row, col, *(uint32_t *)&c_out[row * gemm_l.Np + col], - *(uint32_t *)&gemm_Cref_dram[idx]); - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float actual = partial_at( - partials + (job * kb_count + kb) * PARTIAL_FLOATS, - local_row, local_col); - const float expected = reference_partial(row, col, kb); - printf(" kb=%u actual=0x%08x expected=0x%08x\n", kb, - *(uint32_t *)&actual, *(uint32_t *)&expected); - } - printed_partial_debug = 1; - } - } - } - if (row_errors) - printf("row %u: %u errors, col_mask[63:0]=%08x_%08x\n", - row, row_errors, error_mask_hi, error_mask_lo); - } - - printf("\n=== General FP32 GEMM %ux%ux%u " - "(M64xN64xK64 panels, 2 buffers, 1 core) ===\n", - gemm_l.M, gemm_l.N, gemm_l.K); - printf("Cycles: %u\n", cycles); - printf("Utilization: %u.%02u%%\n", utilization_bp / 100, - utilization_bp % 100); - printf("MaxAbsErr: 0x%08x\n", *(uint32_t *)&local_max_abs); - printf("Errors: %u\n", local_errors); - printf("Status: %s\n", local_errors ? "FAIL" : "PASS"); - return local_errors ? 1 : 0; -} diff --git a/sw/vme/apps/tew32/fp32-gemm-vse/script/gen_data.py b/sw/vme/apps/tew32/fp32-gemm-vse/script/gen_data.py deleted file mode 100644 index 17f3e82b5..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vse/script/gen_data.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 - -# Both store variants consume exactly the same padded and packed inputs. -import pathlib - -exec(compile(open(pathlib.Path(__file__).parents[2] / "fp32-gemm-vtse" / - "script" / "gen_data.py").read(), __file__, "exec")) diff --git a/sw/vme/apps/tew32/fp32-gemm-vtse/CMakeLists.txt b/sw/vme/apps/tew32/fp32-gemm-vtse/CMakeLists.txt deleted file mode 100644 index bb6228ada..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vtse/CMakeLists.txt +++ /dev/null @@ -1,73 +0,0 @@ - -set(VME_GEMM_SHAPES - "64x64x64;64x64x128;64x128x64;64x128x128;128x64x64;128x64x128;128x128x64;128x128x128" CACHE STRING - "Semicolon-separated MxNxK shapes for the general FP32 GEMM tests") - -set(_vme_gemm_build_shapes ${VME_GEMM_SHAPES}) -list(APPEND _vme_gemm_build_shapes - "13x13x13" "13x23x13" "23x13x13" "23x23x13" - "13x45x7" "13x55x7" "23x45x7" "23x55x7" - "45x13x8" "45x23x8" "55x13x8" "55x23x8" - "45x45x9" "45x55x9" "55x45x9" "55x55x9" - "77x13x13" "77x23x13" "87x13x13" "87x23x13" - "77x45x9" "77x55x9" "87x45x9" "87x55x9" - "77x77x77" "77x87x77" "87x77x77" "87x87x77" - "77x109x71" "77x119x71" "87x109x71" "87x119x71" - "109x77x72" "109x87x72" "119x77x72" "119x87x72" - "109x109x73" "109x119x73" "119x109x73" "119x119x73" - "141x77x77" "141x87x77" "151x77x77" "151x87x77" - "141x109x73" "141x119x73" "151x109x73" "151x119x73" - "724x256x192") -set(_vme_gemm_normalized_shapes) -foreach(_shape IN LISTS _vme_gemm_build_shapes) - string(STRIP "${_shape}" _shape) - list(APPEND _vme_gemm_normalized_shapes "${_shape}") -endforeach() -set(_vme_gemm_build_shapes ${_vme_gemm_normalized_shapes}) -list(REMOVE_DUPLICATES _vme_gemm_build_shapes) - -foreach(_shape IN LISTS _vme_gemm_build_shapes) - if(NOT _shape MATCHES "^([0-9]+)x([0-9]+)x([0-9]+)$") - message(FATAL_ERROR "Invalid GEMM shape '${_shape}', expected MxNxK") - endif() - set(_m ${CMAKE_MATCH_1}) - set(_n ${CMAKE_MATCH_2}) - set(_k ${CMAKE_MATCH_3}) - # Keep M/N fixed to the tuned M64xN64 panel and choose the largest useful - # K chunk for this shape. The kernel consumes K in groups of eight and - # requires at least three groups for its prologue/kernel/epilogue schedule. - set(_chunk_n 64) - math(EXPR _chunk_k "((${_k} + 7) / 8) * 8") - if(_chunk_k LESS 24) - set(_chunk_k 24) - elseif(_chunk_k GREATER 128) - set(_chunk_k 128) - endif() - set(_target fp32-gemm-vtse_M${_m}_N${_n}_K${_k}) - set(_generated_dir ${CMAKE_CURRENT_BINARY_DIR}/generated) - set(_header ${_generated_dir}/data/data_${_m}_${_n}_${_k}.h) - - add_custom_command( - OUTPUT ${_header} - COMMAND ${CMAKE_COMMAND} -E make_directory ${_generated_dir}/data - COMMAND ${PYTHON} ${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py - --m ${_m} --n ${_n} --k ${_k} --output ${_header} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/script/gen_data.py - ${CMAKE_CURRENT_SOURCE_DIR}/data/layer.h - VERBATIM) - - if(BUILD_TESTS) - add_custom_target(generate-${_target} DEPENDS ${_header}) - add_snitch_test(${_target} main.c) - add_dependencies(test-${SNITCH_TEST_PREFIX}${_target} generate-${_target}) - target_include_directories(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/data - ${_generated_dir}) - target_compile_definitions(test-${SNITCH_TEST_PREFIX}${_target} PRIVATE - DATAHEADER="data/data_${_m}_${_n}_${_k}.h" - KERNELHEADER="kernel/gemm.c" - GEMM_CHUNK_N=${_chunk_n} - GEMM_CHUNK_K=${_chunk_k}) - endif() -endforeach() diff --git a/sw/vme/apps/tew32/fp32-gemm-vtse/data/layer.h b/sw/vme/apps/tew32/fp32-gemm-vtse/data/layer.h deleted file mode 100644 index 76e859a50..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vtse/data/layer.h +++ /dev/null @@ -1,12 +0,0 @@ - -#include - -typedef struct { - uint32_t M; - uint32_t N; - uint32_t K; - uint32_t Mp; - uint32_t Np; - uint32_t Kp; -} vme_gemm_layer; - diff --git a/sw/vme/apps/tew32/fp32-gemm-vtse/kernel/gemm.c b/sw/vme/apps/tew32/fp32-gemm-vtse/kernel/gemm.c deleted file mode 100644 index 9eb59e951..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vtse/kernel/gemm.c +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 - -#include - -#include "gemm.h" - -enum { - GEMM_TILE_EDGE = 16, - GEMM_PACKED_TILE_EDGE = 16, -}; - -/* - * Execute one tm-by-tn FP32 output tile on mt0. - * - * msetmtypei selects FP32, TWIDEN=1 and therefore LMUL=1 for TE=16. msettn is - * deliberately used for both vector-load lengths: unlike vset[i]vl[i], it - * preserves mtype and keeps the matrix unit configured. - */ -#define GEMM_FP32_TILE(AT_PTR, B_PTR, C_PTR, A_STRIDE, B_STRIDE, C_STRIDE, \ - K_COUNT, TM, TN) \ - do { \ - uintptr_t _at = (uintptr_t)(AT_PTR); \ - uintptr_t _b = (uintptr_t)(B_PTR); \ - uintptr_t _c = (uintptr_t)(C_PTR); \ - uintptr_t _a_stride = (uintptr_t)(A_STRIDE); \ - uintptr_t _b_stride = (uintptr_t)(B_STRIDE); \ - uintptr_t _c_stride = (uintptr_t)(C_STRIDE); \ - uintptr_t _k_count = (uintptr_t)(K_COUNT); \ - uintptr_t _tm = (uintptr_t)(TM); \ - uintptr_t _tn = (uintptr_t)(TN); \ - uintptr_t _tss; \ - uintptr_t _tss_end; \ - \ - asm volatile( \ - "msettm x0, %[tm]\n" \ - "msettn x0, %[tn]\n" \ - "vtzero mt0\n" \ - "beqz %[k_count], .Lgemm_store_%=\n" \ - \ - ".Lgemm_k_%=:\n" \ - "msettn x0, %[tm]\n" \ - "vle32.v v0, (%[at])\n" \ - "add %[at], %[at], %[a_stride]\n" \ - "msettn x0, %[tn]\n" \ - "vle32.v v16, (%[b])\n" \ - "add %[b], %[b], %[b_stride]\n" \ - "vtfmm.tvv mt0, v0, v16\n" \ - "addi %[k_count], %[k_count], -1\n" \ - "bnez %[k_count], .Lgemm_k_%=\n" \ - \ - ".Lgemm_store_%=:\n" \ - "msettn x0, %[tn]\n" \ - "li %[tss], 0\n" \ - "add %[tss_end], %[tss], %[tm]\n" \ - ".Lgemm_store_row_%=:\n" \ - "vtse32 %[tss], (%[c])\n" \ - "add %[c], %[c], %[c_stride]\n" \ - "addi %[tss], %[tss], 1\n" \ - "bltu %[tss], %[tss_end], .Lgemm_store_row_%=\n" \ - : [at] "+&r"(_at), [b] "+&r"(_b), [c] "+&r"(_c), \ - [k_count] "+&r"(_k_count), [tss] "=&r"(_tss), \ - [tss_end] "=&r"(_tss_end) \ - : [a_stride] "r"(_a_stride), [b_stride] "r"(_b_stride), \ - [c_stride] "r"(_c_stride), [tm] "r"(_tm), [tn] "r"(_tn) \ - : "memory"); \ - } while (0) - -static inline void gemm_fp32_configure(void) { - uintptr_t tk = 1; - - asm volatile( - "msetmtypei 1, 2\n" - "msettk x0, %[tk]\n" - : - : [tk] "r"(tk) - : "memory"); -} - -/* - * General, tightly packed ABI: - * addrA = At[K][M], addrB = B[K][N], addrC = C[M][N]. - * - * The physical accumulator has a fixed TE=16. Runtime tm/tn select the - * active rectangle, while each VTFMM consumes one FP32 K row (tk=1) from the - * vectors rooted at v0 and v16. - */ -__attribute__((noinline, aligned(64))) void gemm_fp32( - void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt) -{ - const float *at = (const float *)addrA; - const float *b = (const float *)addrB; - float *c = (float *)addrC; - - if (M <= 0 || N <= 0) - return; - if (K < 0 || alt_fmt != 0) - __builtin_trap(); - - gemm_fp32_configure(); - - for (int m = 0; m < M; m += GEMM_TILE_EDGE) { - const int tm = M - m < GEMM_TILE_EDGE ? M - m : GEMM_TILE_EDGE; - - for (int n = 0; n < N; n += GEMM_TILE_EDGE) { - const int tn = N - n < GEMM_TILE_EDGE ? N - n : GEMM_TILE_EDGE; - - GEMM_FP32_TILE(at + m, b + n, c + (uintptr_t)m * N + n, - (uintptr_t)M * sizeof(float), - (uintptr_t)N * sizeof(float), - (uintptr_t)N * sizeof(float), K, tm, tn); - } - } -} - -/* - * Compatibility entry for the existing DMA driver. Its A/B panels are - * packed as independent K-by-16 tiles, while C remains row-major. This path - * uses the same matrix configuration and the same no-vset microkernel. - */ -__attribute__((noinline, aligned(64))) void gemm_block_fp32( - void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt) -{ - const float *a = (const float *)addrA; - const float *b = (const float *)addrB; - float *c = (float *)addrC; - - if (M <= 0 || N <= 0) - return; - if (K < 0 || alt_fmt != 0) - __builtin_trap(); - - gemm_fp32_configure(); - - for (int m = 0; m < M; m += GEMM_PACKED_TILE_EDGE) { - const int tm = M - m < GEMM_PACKED_TILE_EDGE - ? M - m : GEMM_PACKED_TILE_EDGE; - const float *a_tile = a + (uintptr_t)(m / GEMM_PACKED_TILE_EDGE) * - K * GEMM_PACKED_TILE_EDGE; - - for (int n = 0; n < N; n += GEMM_PACKED_TILE_EDGE) { - const int tn = N - n < GEMM_PACKED_TILE_EDGE - ? N - n : GEMM_PACKED_TILE_EDGE; - const float *b_tile = b + (uintptr_t)(n / GEMM_PACKED_TILE_EDGE) * - K * GEMM_PACKED_TILE_EDGE; - - GEMM_FP32_TILE(a_tile, b_tile, c + (uintptr_t)m * N + n, - GEMM_PACKED_TILE_EDGE * sizeof(float), - GEMM_PACKED_TILE_EDGE * sizeof(float), - (uintptr_t)N * sizeof(float), K, tm, tn); - } - } -} - -#undef GEMM_FP32_TILE diff --git a/sw/vme/apps/tew32/fp32-gemm-vtse/kernel/gemm.h b/sw/vme/apps/tew32/fp32-gemm-vtse/kernel/gemm.h deleted file mode 100644 index 215e6d331..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vtse/kernel/gemm.h +++ /dev/null @@ -1,16 +0,0 @@ - -#include - -static inline uint32_t gemm_cycle(void) { - uint32_t value; - asm volatile("csrr %0, mcycle" : "=r"(value)); - return value; -} - -// General tightly packed kernel: At[K][M], B[K][N], C[M][N]. -void gemm_fp32(void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt); - -// Compatibility entry for the existing packed-panel DMA driver. -void gemm_block_fp32(void *addrA, void *addrB, void *addrC, - int K, int N, int M, int alt_fmt); diff --git a/sw/vme/apps/tew32/fp32-gemm-vtse/main.c b/sw/vme/apps/tew32/fp32-gemm-vtse/main.c deleted file mode 100644 index dd0d464eb..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vtse/main.c +++ /dev/null @@ -1,328 +0,0 @@ -#include -#include - -#include DATAHEADER -#ifdef GEMM_KERNEL_SOURCE -#include GEMM_KERNEL_SOURCE -#else -#include KERNELHEADER -#endif - -enum { - TCDM_BYTES = 256 * 1024, - M_CHUNK = 64, - N_CHUNK = GEMM_CHUNK_N, - K_CHUNK = GEMM_CHUNK_K, - TILE_DIM = 16, - TILE_FLOATS = TILE_DIM * K_CHUNK, - TILE_BYTES = TILE_FLOATS * sizeof(float), - A_TILES = M_CHUNK / TILE_DIM, - B_TILES = N_CHUNK / TILE_DIM, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_BYTES = M_CHUNK * N_CHUNK * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - PARTIAL_FLOATS = M_CHUNK * N_CHUNK, - BUFFER_BYTES = C_OFFSET + C_BYTES, - BUFFER_COUNT = 2, - RESIDENT_A_OFFSET = 0, - RESIDENT_B_OFFSET = RESIDENT_A_OFFSET + A_BYTES, - RESIDENT_C_OFFSET = RESIDENT_B_OFFSET + 2 * B_BYTES, - RESIDENT_BYTES = RESIDENT_C_OFFSET + C_BYTES, -}; - -_Static_assert(BUFFER_BYTES <= TCDM_BYTES, - "selected GEMM chunk does not fit in TCDM"); -_Static_assert(BUFFER_COUNT * BUFFER_BYTES <= TCDM_BYTES, - "GEMM buffer allocation exceeds TCDM"); -_Static_assert(RESIDENT_BYTES <= TCDM_BYTES, - "resident-A, B ping-pong, and C layout exceeds TCDM"); - -static const float zero_tile_dram[TILE_FLOATS] - __attribute__((section(".dram"), aligned(64))) = {0}; - -static uint8_t *l1_base; -static float *partials; -static float *c_out; -static uint32_t allocation_failed; - -static inline uint32_t min_u32(uint32_t a, uint32_t b) { - return a < b ? a : b; -} - -static inline uint32_t float_bits(float value) { - union { - float f; - uint32_t u; - } bits = {.f = value}; - return bits.u; -} - -static inline void finish_spatz(void) { - uint32_t spatz_fcsr; - asm volatile("vtdiscard\n" - "csrr %0, fcsr\n" - "fence rw, rw\n" - : "=&r"(spatz_fcsr) - : - : "memory"); -} - -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t chunk_k, - uint32_t src_tile_k) { - // A panel is physically contiguous only when its source tile stride is - // identical to the local K chunk. For example, Kp=192/K_CHUNK=128 must - // skip over the remaining 64 K rows between packed source tiles. - if (valid_tiles == panel_tiles && chunk_k == K_CHUNK && - src_tile_k == K_CHUNK) - return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - - // D2 first clears every destination tile, then copies only the valid - // K-slices. XDMA executes descriptors in issue order. - const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( - dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); - if (valid_tiles == 0) - return clear_tid; - - const size_t copy_bytes = chunk_k * TILE_DIM * sizeof(float); - const size_t src_stride = src_tile_k * TILE_DIM * sizeof(float); - return snrt_dma_start_2d(dst, src, copy_bytes, TILE_BYTES, src_stride, - valid_tiles); -} - -static snrt_dma_txid_t fill_a(float *a_dst, uint32_t mb, uint32_t kb) { - const uint32_t k0 = kb * K_CHUNK; - const uint32_t chunk_k = min_u32(K_CHUNK, gemm_l.Kp - k0); - const uint32_t mt_count = gemm_l.Mp / TILE_DIM; - const uint32_t first_mt = mb * A_TILES; - const uint32_t valid_tiles = first_mt < mt_count - ? min_u32(A_TILES, mt_count - first_mt) - : 0; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : gemm_Apack_dram + - (first_mt * gemm_l.Kp + k0) * TILE_DIM; - return fill_panel(a_dst, src, A_TILES, valid_tiles, chunk_k, gemm_l.Kp); -} - -static snrt_dma_txid_t fill_b(float *b_dst, uint32_t nb, uint32_t kb) { - const uint32_t k0 = kb * K_CHUNK; - const uint32_t chunk_k = min_u32(K_CHUNK, gemm_l.Kp - k0); - const uint32_t nt_count = gemm_l.Np / TILE_DIM; - const uint32_t first_nt = nb * B_TILES; - const uint32_t valid_tiles = first_nt < nt_count - ? min_u32(B_TILES, nt_count - first_nt) - : 0; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : gemm_Bpack_dram + - (first_nt * gemm_l.Kp + k0) * TILE_DIM; - return fill_panel(b_dst, src, B_TILES, valid_tiles, chunk_k, gemm_l.Kp); -} - -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * BUFFER_BYTES; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); -} - -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { -#ifdef GEMM_PACKED_STORE - const uint32_t tile = (row / TILE_DIM) * B_TILES + col / TILE_DIM; - return partial[tile * TILE_DIM * TILE_DIM + - (row % TILE_DIM) * TILE_DIM + col % TILE_DIM]; -#else - return partial[row * N_CHUNK + col]; -#endif -} - -int main(void) { - const uint32_t cid = snrt_cluster_core_idx(); - if (cid != 0) - return 0; - - const uint32_t row_pairs = M_CHUNK / (2 * TILE_DIM); - const uint32_t mb_count = (gemm_l.Mp + M_CHUNK - 1) / M_CHUNK; - const uint32_t nb_count = (gemm_l.Np + N_CHUNK - 1) / N_CHUNK; - const uint32_t kb_count = (gemm_l.Kp + K_CHUNK - 1) / K_CHUNK; - const uint32_t jobs = mb_count * nb_count; - const uint32_t iterations = jobs * kb_count; - - const uint32_t l1_bytes = BUFFER_COUNT == 2 - ? 2 * BUFFER_BYTES - : RESIDENT_BYTES; - l1_base = (uint8_t *)snrt_l1alloc(l1_bytes); - partials = (float *)snrt_l3alloc(iterations * PARTIAL_FLOATS * - sizeof(float)); - c_out = (float *)snrt_l3alloc(gemm_l.Mp * gemm_l.Np * sizeof(float)); - allocation_failed = l1_base == 0 || partials == 0 || c_out == 0; - if (allocation_failed) { - printf("general GEMM allocation failure\n"); - return 2; - } - - uint32_t measured_cycles = 0; - if (BUFFER_COUNT == 2) { - snrt_dma_txid_t input_tid[2] = {0, 0}; - uint32_t current_buffer = 0; - input_tid[0] = fill_buffer(0, 0, 0, 0); - snrt_dma_wait(input_tid[0]); - for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t next_iter = iter + 1; - const uint32_t has_next = next_iter < iterations; - const uint32_t next_buffer = current_buffer ^ 1; - - if (has_next) { - const uint32_t next_job = next_iter / kb_count; - input_tid[next_buffer] = fill_buffer( - next_buffer, next_job / nb_count, next_job % nb_count, - next_iter % kb_count); - } - uint8_t *base = l1_base + current_buffer * BUFFER_BYTES; - const uint32_t t0 = gemm_cycle(); - for (uint32_t pair = 0; pair < row_pairs; ++pair) { - const uint32_t core_row = pair * 2 * TILE_DIM; - gemm_block_fp32( - (void *)(base + A_OFFSET + - core_row * K_CHUNK * sizeof(float)), - (void *)(base + B_OFFSET), - (void *)(base + C_OFFSET + - core_row * N_CHUNK * sizeof(float)), - (int)K_CHUNK, (int)N_CHUNK, 2 * TILE_DIM, 0); - // Complete this pair's VTSE stream before reusing the tiles. - finish_spatz(); - } - measured_cycles += gemm_cycle() - t0; - - float *src = (float *)(l1_base + - current_buffer * BUFFER_BYTES + C_OFFSET); - snrt_dma_start_1d(partials + iter * PARTIAL_FLOATS, src, C_BYTES); - if (has_next) - snrt_dma_wait(input_tid[next_buffer]); - if (has_next) - current_buffer = next_buffer; - } - } else { - // Retain A while traversing N and prefetch B into the alternate bank. - // C uses one buffer so the 112 KiB layout leaves the per-core stacks - // above TCDM allocation untouched. - float *a = (float *)(l1_base + RESIDENT_A_OFFSET); - float *b[2] = { - (float *)(l1_base + RESIDENT_B_OFFSET), - (float *)(l1_base + RESIDENT_B_OFFSET + B_BYTES), - }; - float *c = (float *)(l1_base + RESIDENT_C_OFFSET); - - for (uint32_t kb = 0; kb < kb_count; ++kb) { - for (uint32_t mb = 0; mb < mb_count; ++mb) { - uint32_t ping = 0; - snrt_dma_wait_all(); - fill_a(a, mb, kb); - fill_b(b[0], 0, kb); - // Make the complete A/B panel visible before compute starts. - snrt_dma_wait_all(); - for (uint32_t nb = 0; nb < nb_count; ++nb) { - const uint32_t has_next_n = nb + 1 < nb_count; - if (has_next_n) - fill_b(b[ping ^ 1], nb + 1, kb); - const uint32_t t0 = gemm_cycle(); - for (uint32_t pair = 0; pair < row_pairs; ++pair) { - const uint32_t core_row = pair * 2 * TILE_DIM; - gemm_block_fp32( - (void *)((uint8_t *)a + - core_row * K_CHUNK * sizeof(float)), - (void *)b[ping], - (void *)((uint8_t *)c + - core_row * N_CHUNK * sizeof(float)), - (int)K_CHUNK, (int)N_CHUNK, - 2 * TILE_DIM, 0); - finish_spatz(); - } - measured_cycles += gemm_cycle() - t0; - - const uint32_t job = mb * nb_count + nb; - const uint32_t partial = job * kb_count + kb; - snrt_dma_start_1d(partials + partial * PARTIAL_FLOATS, - c, C_BYTES); - // C must be copied before the next job overwrites it. - snrt_dma_wait_all(); - ping ^= 1; - } - } - } - } - - snrt_dma_wait_all(); - - const uint32_t output_elements = gemm_l.M * gemm_l.N; - for (uint32_t idx = 0; idx < output_elements; ++idx) { - const uint32_t row = idx / gemm_l.N; - const uint32_t col = idx % gemm_l.N; - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - float sum = 0.0f; - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float *partial = partials + - (job * kb_count + kb) * PARTIAL_FLOATS; - sum += partial_at(partial, local_row, local_col); - } - c_out[row * gemm_l.Np + col] = sum; - } - const uint32_t cycles = measured_cycles; - - enum { CE = 8 }; - const uint64_t useful_fmas = - (uint64_t)gemm_l.M * gemm_l.N * gemm_l.K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - uint32_t local_errors = 0; - float local_max_abs = 0.0f; - const float tolerance = 1.0e-3f; - for (uint32_t row = 0; row < gemm_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < gemm_l.N; ++col) { - const uint32_t idx = row * gemm_l.N + col; - const float output = c_out[row * gemm_l.Np + col]; - const float expected = gemm_Cref_dram[idx]; - if (float_bits(output) == float_bits(expected)) - continue; - float diff = output - expected; - float abs = diff < 0.0f ? -diff : diff; - float ref = expected; - ref = ref < 0.0f ? -ref : ref; - if (abs > local_max_abs) - local_max_abs = abs; - if (abs > tolerance && abs > tolerance * ref) { - ++local_errors; - ++row_errors; - } - } - if (row_errors) - printf("core %u row %u: %u errors\n", cid, row, row_errors); - } - - printf("\n=== General FP32 GEMM %ux%ux%u " - "(M64xN%uxK%u chunks, %u buffer%s, 1 core) ===\n", - gemm_l.M, gemm_l.N, gemm_l.K, N_CHUNK, K_CHUNK, BUFFER_COUNT, - BUFFER_COUNT == 1 ? "" : "s"); - printf("Cycles: %u\n", cycles); - printf("Utilization: %u.%02u%%\n", utilization_bp / 100, - utilization_bp % 100); - printf("MaxAbsErr: 0x%08x\n", *(uint32_t *)&local_max_abs); - printf("Errors: %u\n", local_errors); - printf("Status: %s\n", local_errors ? "FAIL" : "PASS"); - return local_errors ? 1 : 0; -} diff --git a/sw/vme/apps/tew32/fp32-gemm-vtse/script/gen_data.py b/sw/vme/apps/tew32/fp32-gemm-vtse/script/gen_data.py deleted file mode 100644 index a2e437418..000000000 --- a/sw/vme/apps/tew32/fp32-gemm-vtse/script/gen_data.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import pathlib - -import numpy as np - -def align_up(value, alignment): - return (value + alignment - 1) // alignment * alignment - -def c_array(array): - return "{" + ", ".join(repr(float(x)) + "f" for x in array.flat) + "}" - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--m", type=int, required=True) - parser.add_argument("--n", type=int, required=True) - parser.add_argument("--k", type=int, required=True) - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--output", type=pathlib.Path, required=True) - args = parser.parse_args() - - if min(args.m, args.n, args.k) <= 0: - raise ValueError("M, N, and K must all be positive") - - # Only architectural edge tiles are padded. The outer 64x96x64 DMA - # blocking does not expand the complete matrix to a macro-tile boundary. - mp = align_up(args.m, 16) - npad = align_up(args.n, 16) - kp = align_up(args.k, 8) - rng = np.random.default_rng(args.seed) - a = rng.standard_normal((args.m, args.k), dtype=np.float32) - b = rng.standard_normal((args.k, args.n), dtype=np.float32) - c = (a @ b).astype(np.float32) - - ap = np.zeros((mp, kp), dtype=np.float32) - bp = np.zeros((kp, npad), dtype=np.float32) - ap[:args.m, :args.k] = a - bp[:args.k, :args.n] = b - - # One 512-byte LMUL=8 load contains eight K slices of one 16-row/column - # tile. This is the same k-major packing used by the fixed kernels. - apack = ap.reshape(mp // 16, 16, kp // 8, 8).transpose(0, 2, 3, 1).copy() - bpack = bp.reshape(kp // 8, 8, npad // 16, 16).transpose(2, 0, 1, 3).copy() - - text = f'''// Auto-generated by fp32-gemm-vtse/script/gen_data.py. -#include "layer.h" - -static const vme_gemm_layer gemm_l = {{ - .M = {args.m}, .N = {args.n}, .K = {args.k}, - .Mp = {mp}, .Np = {npad}, .Kp = {kp}, -}}; - -static const float gemm_Cref_dram[{args.m * args.n}] - __attribute__((section(".dram"), aligned(64))) = {c_array(c)}; - -static const float gemm_Apack_dram[{mp * kp}] - __attribute__((section(".dram"), aligned(64))) = {c_array(apack)}; - -static const float gemm_Bpack_dram[{kp * npad}] - __attribute__((section(".dram"), aligned(64))) = {c_array(bpack)}; -''' - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(text) - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/data/data_64_64_64.h b/sw/vme/apps/tew32/fp32-matmul-vle-vse/data/data_64_64_64.h deleted file mode 100644 index 49d1b3900..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/data/data_64_64_64.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Auto-generated by script/gen_data.py — do not edit by hand. - -#include "layer.h" - -const vme_gemm_layer gemm_l = { - .M = 64, - .N = 64, - .K = 64, - .TM = 8, - .TN = 8, -}; - -// C[M*N] row-major fp32 golden reference, C = A*B -static float gemm_C_dram[64*64] __attribute__((section(".data"))) = -{11.479634284973145f, -6.046323299407959f, -4.815614700317383f, -2.040816068649292f, -8.940674781799316f, 1.9331291913986206f, 4.764830112457275f, 14.248414993286133f, -3.9397127628326416f, -2.1485483646392822f, -0.8390139937400818f, -1.3708610534667969f, 13.092900276184082f, 3.2064008712768555f, 5.635993480682373f, 0.7455204725265503f, 7.919027805328369f, -11.349638938903809f, -4.450965404510498f, 15.0212984085083f, 0.6696339845657349f, -0.1861145943403244f, 4.869348049163818f, 3.4079155921936035f, -3.9942312240600586f, 8.851547241210938f, 8.788155555725098f, 6.571914127562195e-05f, 9.69917106628418f, -8.461928367614746f, -8.80200481414795f, 4.7893218994140625f, -0.18487417697906494f, -11.189379692077637f, -12.456897735595703f, -3.528578758239746f, 6.636075019836426f, 4.215647220611572f, -2.5566506385803223f, -2.2709872722625732f, 24.085134506225586f, -3.519670009613037f, -1.5794446468353271f, 13.206070899963379f, -9.0339937210083f, -0.49932846426963806f, 8.33430290222168f, -3.4920732975006104f, 24.13520622253418f, -2.866286516189575f, -5.177119255065918f, 8.296358108520508f, -2.486847162246704f, 7.566592693328857f, -18.43720817565918f, 17.894702911376953f, -17.132667541503906f, 4.636127471923828f, -15.027854919433594f, 14.163813591003418f, -16.567644119262695f, 4.0814080238342285f, -19.760257720947266f, 0.5436081290245056f, -1.0178847312927246f, -0.959050178527832f, -7.164222717285156f, 0.9985886216163635f, -8.26685619354248f, -2.2400271892547607f, 0.6488896608352661f, -3.7606098651885986f, 6.696377277374268f, 1.2368918657302856f, 0.3883592486381531f, -2.765044689178467f, 1.5367969274520874f, 5.624364852905273f, -1.1590087413787842f, 5.226413249969482f, 8.801167488098145f, 0.28941860795021057f, 7.781001091003418f, 1.1585612297058105f, -15.459006309509277f, -3.4766643047332764f, -8.453717231750488f, 10.173603057861328f, 3.6514530181884766f, 1.6193315982818604f, 5.127340793609619f, -0.9438349008560181f, -3.7507288455963135f, 13.667619705200195f, -10.900826454162598f, 11.15420913696289f, -2.949625253677368f, -2.746349811553955f, -16.438350677490234f, -17.064266204833984f, -4.720984935760498f, 4.893701553344727f, -2.473956823348999f, 4.502293586730957f, -5.264101982116699f, -8.454588890075684f, -5.186229705810547f, 5.584354877471924f, -2.95721435546875f, -3.410273790359497f, -6.588162899017334f, -0.8277148604393005f, -2.7242908477783203f, 4.025758266448975f, -1.4152922630310059f, 2.386408805847168f, 7.04514741897583f, -0.22987337410449982f, -0.45179229974746704f, 12.081316947937012f, -5.206918716430664f, 5.094143390655518f, -5.665627956390381f, 10.579627990722656f, -6.4196953773498535f, -8.07258415222168f, -19.385873794555664f, -8.815452575683594f, 1.0405519008636475f, -15.006485939025879f, -7.755399227142334f, 10.724888801574707f, 6.764276504516602f, 8.178961753845215f, 7.572741985321045f, 18.397531509399414f, -1.7755249738693237f, -1.9764132499694824f, 2.622986078262329f, -3.577617645263672f, -0.9461591839790344f, 14.711101531982422f, 8.183825492858887f, -7.469543933868408f, 14.31879711151123f, -6.517776012420654f, 2.1930079460144043f, 6.8929572105407715f, -10.022858619689941f, -1.7278424501419067f, 12.308111190795898f, 13.900723457336426f, 7.140420913696289f, 12.356964111328125f, 4.42258358001709f, -0.5004820227622986f, -1.5587430000305176f, 15.559115409851074f, 0.4262810945510864f, -2.8206470012664795f, 15.083824157714844f, -4.9072794914245605f, 2.5857160091400146f, 4.584408283233643f, -17.988496780395508f, -11.013274192810059f, 2.8712387084960938f, 4.990177631378174f, 0.6336867213249207f, 8.750150680541992f, 18.924428939819336f, 3.4662630558013916f, -14.959811210632324f, 3.8478240966796875f, 17.260263442993164f, -13.368204116821289f, 2.653244972229004f, -4.103200435638428f, -12.37991714477539f, -0.2966856062412262f, 5.892507076263428f, 6.881699562072754f, -3.230393648147583f, -10.482145309448242f, -13.298931121826172f, -6.5993547439575195f, 0.7129412293434143f, -2.0269880294799805f, 4.382270336151123f, -6.041990756988525f, -17.212663650512695f, -6.112694263458252f, 13.97390365600586f, -4.114153861999512f, -0.9799354672431946f, -4.399726390838623f, -5.032261848449707f, 0.5352955460548401f, -1.2309777736663818f, 10.451981544494629f, -5.048009872436523f, 3.677152633666992f, 11.969364166259766f, -0.417408287525177f, -3.176297426223755f, 5.826728343963623f, -5.306847095489502f, 10.880280494689941f, 12.627052307128906f, -5.021015644073486f, 0.382880836725235f, -12.032983779907227f, -2.4724607467651367f, -2.954784393310547f, 4.853247165679932f, -2.217081308364868f, -7.221776485443115f, 5.463408470153809f, -0.44735994935035706f, -10.677308082580566f, 1.9355524778366089f, -2.376770496368408f, 1.6620814800262451f, -12.09306526184082f, 9.559584617614746f, -1.3697234392166138f, -8.889398574829102f, 13.613365173339844f, -6.308327674865723f, -2.4649970531463623f, 2.388371467590332f, 4.159750938415527f, 0.5139855742454529f, 4.015327453613281f, -3.1259841918945312f, -1.0106412172317505f, 3.4432032108306885f, 4.1760029792785645f, -6.856860637664795f, -2.4840879440307617f, 0.686843752861023f, -1.0297985076904297f, 2.4013655185699463f, -3.5610408782958984f, 0.8839467167854309f, -2.944474458694458f, 12.640097618103027f, -3.0358707904815674f, -8.915057182312012f, 20.95890998840332f, 11.799171447753906f, 0.5034058690071106f, -1.9294973611831665f, -1.5430312156677246f, -1.1323392391204834f, 10.141955375671387f, -0.7313078045845032f, 3.43355655670166f, 12.49581241607666f, -0.004268331453204155f, 10.47712230682373f, 19.086078643798828f, -5.440404891967773f, -6.789121150970459f, 0.7849673628807068f, -0.7528467178344727f, -3.794801712036133f, -7.670317649841309f, 14.326471328735352f, -9.476102828979492f, 6.799417972564697f, -10.577319145202637f, -12.32802963256836f, 3.4096364974975586f, -24.734689712524414f, 3.53064227104187f, -1.0564911365509033f, 1.1934951543807983f, 2.728233814239502f, -2.273710012435913f, 1.6628460884094238f, 2.3249359130859375f, 3.8170626163482666f, 4.500376224517822f, -10.710893630981445f, 11.639716148376465f, -4.631977558135986f, 14.680305480957031f, 4.689661979675293f, -21.389917373657227f, 5.577933311462402f, 10.157344818115234f, 1.3816673755645752f, -9.77694034576416f, -1.59405517578125f, -5.5923357009887695f, 4.891682147979736f, -7.728332996368408f, -5.265920639038086f, 8.699220657348633f, -16.415210723876953f, 15.636664390563965f, -9.588492393493652f, -0.5692959427833557f, 2.7607104778289795f, 1.7100213766098022f, 5.899023532867432f, 6.637477874755859f, 1.3234360218048096f, 11.593015670776367f, 3.8447062969207764f, -12.39452075958252f, 17.31923484802246f, -4.310085296630859f, 1.3943078517913818f, 3.690248966217041f, 1.5341496467590332f, -0.16133688390254974f, -5.661871910095215f, -4.113314628601074f, 6.673490524291992f, -28.29741668701172f, -3.9005439281463623f, 5.377161979675293f, 2.418168067932129f, 14.741388320922852f, 12.219460487365723f, 2.1486709117889404f, -0.9644702672958374f, 2.6650185585021973f, -5.7412190437316895f, -11.613107681274414f, -4.728487014770508f, 4.203025817871094f, 4.285702705383301f, -6.733584880828857f, -1.751605749130249f, 1.8439090251922607f, 3.607456684112549f, -7.146326541900635f, 10.433805465698242f, 2.1755292415618896f, 7.192582607269287f, 2.093909502029419f, -12.907740592956543f, 10.749999046325684f, -0.37183743715286255f, -13.879500389099121f, -2.7690041065216064f, -0.5802304744720459f, -2.454869508743286f, 6.290034770965576f, -1.3150807619094849f, 1.3150523900985718f, -7.275416851043701f, -4.452425479888916f, 13.685830116271973f, -10.66769790649414f, 1.0413782596588135f, -11.05685043334961f, 18.703815460205078f, -1.3163670301437378f, 4.386673450469971f, 6.897695541381836f, -10.802721977233887f, -1.752117395401001f, 5.392110824584961f, -6.665993690490723f, 6.647964000701904f, 6.248779296875f, -8.761367797851562f, 3.014192581176758f, -12.200126647949219f, -4.44820499420166f, -6.547129154205322f, 5.838053226470947f, -3.972179889678955f, 4.548210620880127f, -0.38630470633506775f, 0.557646632194519f, -7.725645065307617f, -3.0316765308380127f, -10.36661148071289f, 9.558512687683105f, 1.3280283212661743f, -0.6749752759933472f, 2.5804808139801025f, 14.323273658752441f, 3.224461317062378f, -11.820540428161621f, -3.7925498485565186f, -8.334758758544922f, 0.4022352397441864f, 7.823636531829834f, -3.6518750190734863f, 2.0640387535095215f, -2.0144593715667725f, 1.261433482170105f, -12.39039421081543f, 1.648199200630188f, -8.816394805908203f, -6.666141986846924f, 4.299948692321777f, 11.934805870056152f, -9.07494831085205f, 8.816624641418457f, -7.657290935516357f, -1.1243833303451538f, 3.259331464767456f, 2.445486545562744f, 0.27485233545303345f, 9.906669616699219f, 10.861600875854492f, 1.530747413635254f, 3.6094765663146973f, 6.081133842468262f, -6.803634166717529f, 9.30950927734375f, -7.500870704650879f, -0.20945091545581818f, -4.56763219833374f, 9.062397956848145f, -3.900595188140869f, 0.46193817257881165f, -4.673060417175293f, -0.03734937310218811f, -7.745742321014404f, -1.5532246828079224f, -0.17770543694496155f, 7.410264492034912f, -2.503795862197876f, -4.5144267082214355f, 1.2131376266479492f, 1.3187705278396606f, -1.5181187391281128f, 1.0714176893234253f, 8.04094409942627f, 0.9295154213905334f, 4.887729167938232f, 4.386360168457031f, 2.0568528175354004f, -11.794127464294434f, -8.06806755065918f, 1.753397822380066f, -17.30348777770996f, -9.977322578430176f, 1.4055453538894653f, -3.8515894412994385f, -5.236893653869629f, 14.334654808044434f, -2.8135969638824463f, 12.197455406188965f, 5.414585113525391f, -3.6971359252929688f, 3.15084171295166f, -5.809675216674805f, 1.7624473571777344f, -1.7521796226501465f, -4.359520435333252f, -6.7524309158325195f, -6.430643558502197f, 4.465829372406006f, 5.363595008850098f, -0.05738918483257294f, -1.2110704183578491f, -12.731424331665039f, 7.597854137420654f, 5.500415325164795f, -6.625348091125488f, -4.435070991516113f, 16.017005920410156f, 6.0957231521606445f, 3.6745896339416504f, 18.56254005432129f, -6.415927886962891f, 7.9389967918396f, 2.5001425743103027f, 13.855697631835938f, 3.3746798038482666f, -13.604127883911133f, 0.829696774482727f, -0.6029493808746338f, 5.70452880859375f, 2.657468795776367f, -2.0609846115112305f, -5.867642402648926f, 9.778928756713867f, 4.533025741577148f, 5.282003402709961f, -1.0826342105865479f, -6.271111965179443f, 0.7187356352806091f, 8.167415618896484f, -17.741748809814453f, 5.170361518859863f, 1.9452539682388306f, 0.710178554058075f, 15.03416633605957f, 5.8055901527404785f, 9.560342788696289f, -7.244373798370361f, -7.103370189666748f, 6.701330184936523f, -13.6785306930542f, -7.435286998748779f, -1.4204037189483643f, 3.6168243885040283f, -1.2848598957061768f, 10.019153594970703f, -5.542983531951904f, 7.791118621826172f, 10.698689460754395f, 0.5794566869735718f, -5.135282039642334f, -0.725027859210968f, 6.094788551330566f, 3.9960646629333496f, 9.213926315307617f, -1.1690841913223267f, -11.310675621032715f, 4.797519207000732f, 2.9180209636688232f, 17.760066986083984f, -3.3779516220092773f, -5.8081278800964355f, 11.280855178833008f, -4.242335319519043f, -9.20323657989502f, 5.160378456115723f, 3.6303231716156006f, 8.061834335327148f, 0.7347631454467773f, 10.492819786071777f, -0.8277621865272522f, 15.720081329345703f, 6.086870193481445f, 2.3116986751556396f, -8.257061958312988f, -11.12254524230957f, -4.629530906677246f, -1.0439260005950928f, 2.503861665725708f, 4.953763484954834f, -7.421053886413574f, -4.31266975402832f, -0.7821304202079773f, -9.519991874694824f, -5.7066450119018555f, 2.1456873416900635f, 0.6137762665748596f, 9.2358980178833f, -4.393798351287842f, 0.7774427533149719f, -6.750569820404053f, 6.697755813598633f, 1.7473117113113403f, -5.084836959838867f, -0.3822762370109558f, 3.1108381748199463f, 1.4163764715194702f, -2.868486166000366f, -2.0308327674865723f, 12.085240364074707f, 0.32344046235084534f, -8.123509407043457f, -8.63557243347168f, 0.37889865040779114f, 5.865474700927734f, -2.006664752960205f, -4.2740278244018555f, -3.2797470092773438f, 0.7500452995300293f, 6.715641021728516f, 1.116511583328247f, -5.29896879196167f, 15.738234519958496f, -1.072593331336975f, 7.1444854736328125f, -0.17794132232666016f, 0.2486240416765213f, 10.500697135925293f, 5.418370246887207f, -2.850708484649658f, 5.028833866119385f, -1.3049635887145996f, -3.1064016819000244f, 7.260385036468506f, -0.2315407395362854f, 13.63786506652832f, -3.968128204345703f, 5.425351619720459f, 5.989855766296387f, 0.4256654977798462f, -11.24907112121582f, -13.982461929321289f, 3.0191447734832764f, -2.6573190689086914f, 0.18543148040771484f, 2.4465410709381104f, 7.224967002868652f, 9.639179229736328f, 11.72110366821289f, -14.941984176635742f, 2.158461809158325f, -9.967697143554688f, 7.731217384338379f, 0.2547948360443115f, 8.561541557312012f, -1.320813775062561f, -6.716103553771973f, -13.876091003417969f, -10.98348331451416f, 4.823574066162109f, 5.995814323425293f, 13.67184829711914f, -3.9956705570220947f, 2.217562675476074f, 8.181840896606445f, 11.416173934936523f, -7.419627666473389f, -2.8480899333953857f, -3.8081743717193604f, -9.326810836791992f, 2.7183923721313477f, 3.146589994430542f, -3.0829880237579346f, -5.814912796020508f, 8.27266788482666f, -2.7244033813476562f, 1.7461737394332886f, 12.851394653320312f, 6.015487194061279f, -11.382539749145508f, -10.727778434753418f, 1.8263652324676514f, 4.965327262878418f, -4.425846576690674f, 1.1692211627960205f, 0.7969360947608948f, 0.08354203402996063f, 6.083127975463867f, 5.9863152503967285f, -6.197132587432861f, -0.453395277261734f, -1.288467288017273f, -6.85847806930542f, 6.487435340881348f, -5.534632682800293f, 13.624343872070312f, 1.5271718502044678f, -1.4509201049804688f, -12.175165176391602f, -9.207465171813965f, 9.6865873336792f, -4.8576860427856445f, 5.351866245269775f, -8.007092475891113f, -1.1575658321380615f, 13.337957382202148f, 11.559327125549316f, -15.20022201538086f, 0.6360299587249756f, -16.20244789123535f, 7.8914031982421875f, 7.922462463378906f, 0.49626535177230835f, 0.6453051567077637f, 17.201900482177734f, 4.8184661865234375f, -9.721735954284668f, -9.466944694519043f, -1.581121563911438f, -3.897400379180908f, 13.486580848693848f, -2.706510305404663f, 2.2266478538513184f, -10.676257133483887f, -8.26251220703125f, -3.0972604751586914f, 6.262456893920898f, -7.870155334472656f, -13.106574058532715f, 1.9502202272415161f, 7.944813251495361f, 12.406501770019531f, -14.935750007629395f, 14.834214210510254f, -6.443024635314941f, -7.034519195556641f, 19.392358779907227f, -6.4235076904296875f, 5.952810287475586f, 4.202585697174072f, -6.166481018066406f, 6.54275369644165f, 4.269073009490967f, -4.20452356338501f, -11.693081855773926f, 0.5500400066375732f, -6.326947212219238f, -8.8986234664917f, -15.315683364868164f, 11.211211204528809f, -3.6820297241210938f, -4.241352081298828f, -0.7907126545906067f, 9.467594146728516f, 3.63688588142395f, -3.874994993209839f, -9.040021896362305f, 0.5441036224365234f, 14.560470581054688f, -4.352148532867432f, 2.122553586959839f, 7.5770955085754395f, 10.166125297546387f, -2.813676118850708f, -5.165591716766357f, -10.61868953704834f, 2.6967086791992188f, 1.273497462272644f, 16.388473510742188f, 12.926139831542969f, -18.8586483001709f, -2.5625576972961426f, -1.4167602062225342f, 3.1684563159942627f, -0.9735271334648132f, -12.724577903747559f, -9.471477508544922f, 1.9890105724334717f, -5.327760696411133f, 4.15165901184082f, -0.2690247595310211f, -5.091151237487793f, -8.251236915588379f, -13.286526679992676f, 11.11539363861084f, -2.1142094135284424f, -8.37338924407959f, -4.229764461517334f, 2.3841516971588135f, 9.676994323730469f, -16.521343231201172f, 21.009220123291016f, -14.100972175598145f, -18.230031967163086f, -2.3318724632263184f, 6.4106621742248535f, 5.1233930587768555f, 3.14448618888855f, -2.510714292526245f, 4.122291088104248f, -16.625221252441406f, -0.8044806122779846f, -7.060884952545166f, 15.047514915466309f, 1.5442057847976685f, -1.132941484451294f, 1.3845758438110352f, 5.088216781616211f, 5.57691764831543f, -5.411052703857422f, 15.526586532592773f, -8.33289623260498f, 2.9822065830230713f, -1.6032358407974243f, -16.77972984313965f, -8.319680213928223f, 1.4303581714630127f, 16.107824325561523f, -0.7335758805274963f, -10.483888626098633f, 6.441460132598877f, 4.506333827972412f, 2.4685215950012207f, 4.076876640319824f, -11.192659378051758f, -4.523487091064453f, -7.707409858703613f, 11.638382911682129f, -8.234586715698242f, 3.4906229972839355f, -9.156494140625f, -2.1987483501434326f, -14.089797019958496f, -0.26744771003723145f, 9.441929817199707f, 9.555951118469238f, 3.5714211463928223f, -13.786009788513184f, 2.148815870285034f, -6.62591028213501f, 7.7097487449646f, -1.0936174392700195f, 6.26248836517334f, 4.983580589294434f, -14.675965309143066f, -6.113205909729004f, 6.947558879852295f, -8.564501762390137f, 12.769579887390137f, -2.9595448970794678f, -2.199409008026123f, 4.292818069458008f, 0.014189623296260834f, -13.7341890335083f, 9.055596351623535f, 9.658831596374512f, 11.737518310546875f, -1.8660013675689697f, -7.643200397491455f, -7.508113384246826f, 8.38414192199707f, 5.401965618133545f, 13.07395076751709f, 3.3191146850585938f, 4.307673931121826f, 2.2782421112060547f, 1.9879189729690552f, -8.233000755310059f, -9.047646522521973f, 7.657324314117432f, 4.247537612915039f, 11.054131507873535f, -4.79965353012085f, -0.3028814196586609f, 3.658170461654663f, 6.45146369934082f, -2.8840818405151367f, 2.958085536956787f, 0.6611911654472351f, -15.455345153808594f, 14.44206428527832f, -1.5460273027420044f, 7.024033546447754f, -4.981692790985107f, -1.476488709449768f, -11.16106128692627f, 5.7634100914001465f, -4.065712928771973f, -7.169682502746582f, 6.687571048736572f, 9.344539642333984f, 0.3659030795097351f, 9.137345314025879f, -2.647665023803711f, -1.841318964958191f, 5.127373695373535f, -10.907052993774414f, 12.13635540008545f, 7.434513568878174f, -4.468003273010254f, 3.7723612785339355f, 6.410868167877197f, 4.488480091094971f, -4.279177665710449f, 7.486018180847168f, 2.041238784790039f, 0.8295110464096069f, 6.553818702697754f, 0.889700710773468f, 4.766284465789795f, -10.556770324707031f, -4.710302352905273f, -4.570557594299316f, 2.7720491886138916f, -0.9912987351417542f, -14.845259666442871f, 9.316060066223145f, 1.0089187622070312f, 8.025344848632812f, -1.9029171466827393f, 7.934247016906738f, -9.479381561279297f, 2.46075439453125f, 4.621790409088135f, -9.061838150024414f, 5.576177597045898f, 1.5385044813156128f, -0.02164754830300808f, 3.5387630462646484f, 8.792508125305176f, 11.135085105895996f, 0.09514058381319046f, 2.4890284538269043f, 6.469166278839111f, -12.828638076782227f, -11.913107872009277f, -12.835579872131348f, -0.7011198997497559f, 7.332045555114746f, 8.766676902770996f, -5.067351341247559f, -0.9870837926864624f, -12.046849250793457f, -8.263047218322754f, 7.80837869644165f, -2.4639174938201904f, 4.959366798400879f, 1.3135488033294678f, 0.6050440073013306f, 4.289178848266602f, -1.3765535354614258f, -9.210429191589355f, -2.441307544708252f, -13.704766273498535f, 1.7074304819107056f, 8.250596046447754f, -9.176819801330566f, -2.5442323684692383f, -10.301264762878418f, -9.052530288696289f, -10.033407211303711f, -6.968941688537598f, -2.9384915828704834f, 11.104891777038574f, 12.18455696105957f, -1.7053841352462769f, -10.236660957336426f, 13.148673057556152f, 5.770162105560303f, -14.608895301818848f, -3.367353677749634f, -1.997019648551941f, 0.6409828662872314f, 10.021282196044922f, 5.919330596923828f, -1.6916216611862183f, 8.11469841003418f, 15.174899101257324f, 1.4809973239898682f, 7.514713764190674f, -5.768565654754639f, 2.8586220741271973f, 0.785350501537323f, 6.117953300476074f, 12.343249320983887f, 11.014166831970215f, -3.0343480110168457f, -3.784940004348755f, -0.5094264149665833f, -3.813234567642212f, -1.6921602487564087f, 9.909290313720703f, 4.26685905456543f, -0.35572367906570435f, 13.063310623168945f, 0.9254683256149292f, -0.6383225321769714f, -4.169843673706055f, -0.2921450138092041f, -9.779097557067871f, 0.32207998633384705f, 0.46088171005249023f, -1.499398112297058f, -11.92739486694336f, 6.62190055847168f, 16.76767349243164f, 7.850483417510986f, -6.002956867218018f, -6.0743865966796875f, 8.590424537658691f, -8.350720405578613f, -7.048295497894287f, 8.3378267288208f, 10.439396858215332f, 4.051339149475098f, 7.447882652282715f, 2.115618944168091f, -10.832535743713379f, 3.544856071472168f, -10.704363822937012f, -1.747441053390503f, 3.1754229068756104f, -4.96966028213501f, -10.882390975952148f, 2.8613874912261963f, -14.762873649597168f, -3.866286516189575f, -9.634818077087402f, -4.488661289215088f, -16.146400451660156f, -8.561323165893555f, 10.843777656555176f, 7.5432963371276855f, 8.12264347076416f, -7.4483113288879395f, -10.192276000976562f, -0.18046212196350098f, 11.086979866027832f, 28.458358764648438f, -5.693641662597656f, 6.983304500579834f, 4.484742164611816f, -7.9314494132995605f, 5.449891567230225f, 3.5121002197265625f, -10.800859451293945f, 2.2634434700012207f, -10.603031158447266f, 6.369093894958496f, 21.34639549255371f, 5.682496070861816f, -6.500676155090332f, -3.933079719543457f, 9.044882774353027f, -1.2291513681411743f, 2.2405006885528564f, 7.146441459655762f, -5.252052307128906f, 13.324518203735352f, -2.54026198387146f, -4.696802139282227f, -1.9487093687057495f, 1.8994616270065308f, -12.246219635009766f, -6.6798505783081055f, -7.5826520919799805f, -4.574042320251465f, -1.7202016115188599f, 2.292769193649292f, 10.378962516784668f, 6.300224781036377f, 6.089559555053711f, 2.338552713394165f, 10.752300262451172f, -9.21757984161377f, 3.5770480632781982f, 1.6553243398666382f, -5.025013446807861f, -1.0492916107177734f, 2.3430588245391846f, -2.3533082008361816f, 13.465797424316406f, 8.130950927734375f, -17.104875564575195f, 3.76094913482666f, 3.0935189723968506f, 1.6557022333145142f, -1.1840074062347412f, 13.991741180419922f, 18.070926666259766f, -1.3191888332366943f, -5.403342247009277f, 3.2503833770751953f, 11.285150527954102f, 9.816794395446777f, -5.216005802154541f, -1.5604654550552368f, 6.930087089538574f, 2.469757556915283f, 0.5377483367919922f, -6.6069655418396f, -2.20501971244812f, -7.6572394371032715f, -12.394196510314941f, 4.9394073486328125f, -18.88083267211914f, 16.13848114013672f, 2.5295422077178955f, -6.287371635437012f, 3.2773492336273193f, 3.9482178688049316f, 1.5670056343078613f, -0.5477782487869263f, -9.703927993774414f, 7.892303466796875f, 16.467126846313477f, 1.2177623510360718f, -3.8086254596710205f, -0.10376394540071487f, 2.0969502925872803f, -13.377406120300293f, 11.43333625793457f, 0.33095118403434753f, -4.543916702270508f, -8.336254119873047f, 10.293583869934082f, -5.526627063751221f, 7.208284854888916f, 1.3287492990493774f, -0.00872433464974165f, -4.405787467956543f, 24.910831451416016f, 5.813371658325195f, -2.3747384548187256f, -1.58162522315979f, 7.138689041137695f, -0.8299044370651245f, 1.7968316078186035f, -0.16036289930343628f, 7.750216007232666f, -7.1681060791015625f, -1.6841415166854858f, 8.755945205688477f, -6.4495062828063965f, -3.929804801940918f, -11.690154075622559f, 5.407276630401611f, -19.410442352294922f, -7.46274471282959f, -6.528491497039795f, 11.476672172546387f, -10.896162986755371f, 9.078445434570312f, 4.966217994689941f, 2.7781200408935547f, 8.466453552246094f, 6.95072078704834f, -14.296183586120605f, 5.738059997558594f, -0.7061849236488342f, -3.4425599575042725f, -0.2813775837421417f, 4.374084949493408f, -2.8933258056640625f, -10.747404098510742f, -2.676332473754883f, 3.489074230194092f, 9.93961238861084f, 5.222956657409668f, -17.090818405151367f, 1.7347217798233032f, 13.138232231140137f, 8.740431785583496f, 6.423698425292969f, 1.796844244003296f, 2.321765661239624f, 6.327892303466797f, 0.3434549570083618f, 4.261251926422119f, 8.633129119873047f, -9.238373756408691f, 2.2492949962615967f, 2.5688412189483643f, -10.994841575622559f, -2.4493515491485596f, -3.7998733520507812f, -7.384821891784668f, -3.2261850833892822f, 17.528900146484375f, 3.3615753650665283f, 9.745475769042969f, -8.878006935119629f, 5.870270252227783f, -3.883683919906616f, -0.0326266773045063f, 7.409327030181885f, 10.064127922058105f, 17.231494903564453f, -4.299467086791992f, 0.6162081956863403f, -3.9329636096954346f, 3.457488775253296f, 1.6008586883544922f, 14.369483947753906f, -7.038842678070068f, 2.9360132217407227f, -11.51530933380127f, -6.849390029907227f, 13.802511215209961f, -3.17645263671875f, 8.121315002441406f, -4.379727840423584f, -0.7692107558250427f, -7.983859539031982f, -2.9673516750335693f, -16.916322708129883f, -16.98297882080078f, -10.993241310119629f, 6.7799153327941895f, -8.828645706176758f, -8.281076431274414f, 0.38319116830825806f, -6.442864418029785f, 5.842170715332031f, 2.7757022380828857f, -0.3426856994628906f, 11.318920135498047f, 9.262431144714355f, -17.823097229003906f, -11.620147705078125f, 0.7059615254402161f, 4.266175746917725f, 13.950201988220215f, -9.230637550354004f, -0.817978024482727f, 7.5885210037231445f, 15.164800643920898f, 7.9170756340026855f, -8.626565933227539f, -11.744421005249023f, -8.501404762268066f, -5.230059623718262f, -4.577203273773193f, -2.578476667404175f, 4.45864725112915f, 11.191590309143066f, 0.7849674224853516f, 2.8678131103515625f, 19.044151306152344f, -5.121172904968262f, 1.8655592203140259f, 5.731597900390625f, -4.264453411102295f, 0.31350159645080566f, 16.386018753051758f, -9.369375228881836f, -7.727303981781006f, -5.3810529708862305f, 4.404619216918945f, -5.699888229370117f, -5.798112869262695f, 0.13061971962451935f, 2.239311456680298f, 16.179100036621094f, -7.417810440063477f, 2.5456221103668213f, -3.236337900161743f, 1.1458196640014648f, -0.1307709664106369f, -0.8587997555732727f, -4.094841480255127f, -7.875479698181152f, -2.913313865661621f, 3.096707344055176f, 1.1844000816345215f, 11.496950149536133f, 4.843856334686279f, -4.238936424255371f, -1.1538751125335693f, -3.270214080810547f, 11.011395454406738f, 6.255121231079102f, 2.79670786857605f, -2.360741376876831f, 6.709340572357178f, -0.9215971827507019f, 4.753165245056152f, 11.404985427856445f, -11.04754638671875f, 5.086482048034668f, -1.1718279123306274f, 1.6178005933761597f, -1.7486406564712524f, -2.919844627380371f, 7.490941047668457f, -0.875328004360199f, -5.549253940582275f, -8.682398796081543f, 12.354720115661621f, -6.518106937408447f, 3.3926303386688232f, -7.628448486328125f, -0.5480865240097046f, 2.757507085800171f, -5.0333638191223145f, -12.717732429504395f, 9.747604370117188f, -14.137182235717773f, 13.830326080322266f, -8.072053909301758f, 9.366421699523926f, 4.771058082580566f, -3.8767879009246826f, 12.098913192749023f, -1.3983639478683472f, -2.516928195953369f, 17.085342407226562f, 0.700747013092041f, -4.593225479125977f, 3.2615785598754883f, 4.119733810424805f, 8.652438163757324f, 7.159962177276611f, -6.102161407470703f, -7.0815653800964355f, -4.8683037757873535f, -8.63952350616455f, -3.9259283542633057f, 9.596209526062012f, -9.913541793823242f, 3.975876569747925f, -3.891753911972046f, 5.650779724121094f, -6.815282344818115f, 2.731234550476074f, -0.2151940017938614f, -4.856814384460449f, 0.45669957995414734f, -7.445348262786865f, 9.566651344299316f, 1.5104864835739136f, 11.227133750915527f, -0.11537183076143265f, 4.145509719848633f, 4.6820244789123535f, 6.9148335456848145f, 4.9869384765625f, 6.348864555358887f, 0.12999588251113892f, -12.564311981201172f, -4.362220764160156f, -2.023157835006714f, -5.578927516937256f, 10.822314262390137f, -8.657614707946777f, 11.476557731628418f, -7.976850509643555f, 8.056036949157715f, 3.8919501304626465f, -3.0189290046691895f, 0.370013564825058f, 7.211360454559326f, 6.979368209838867f, -5.752582550048828f, -18.53067398071289f, -0.06240836903452873f, 6.615059852600098f, 14.322349548339844f, 7.855193614959717f, -2.3387138843536377f, -7.839303970336914f, -1.904661774635315f, 1.820509672164917f, 0.5016192197799683f, -14.7440767288208f, 13.704774856567383f, 11.45584774017334f, 10.099801063537598f, 0.022373056039214134f, -2.2562215328216553f, -3.3610339164733887f, -18.4713134765625f, -10.077775955200195f, 8.24351978302002f, -11.9453125f, 1.6951130628585815f, -6.876453399658203f, -9.18753433227539f, -3.261322498321533f, -5.389860153198242f, 1.553828239440918f, -3.9102296829223633f, 6.239824295043945f, 0.8332391381263733f, -0.4328247308731079f, -8.079927444458008f, 3.283322334289551f, 2.487851619720459f, 3.38568115234375f, 1.1550346612930298f, 5.795931339263916f, -3.827712059020996f, -17.76996612548828f, -1.3455630540847778f, -8.596356391906738f, -1.9443339109420776f, -3.347691535949707f, -3.8022572994232178f, -1.9706828594207764f, -8.254030227661133f, 4.786689758300781f, -9.022886276245117f, -10.170120239257812f, -7.9390668869018555f, -5.307765960693359f, -1.4348095655441284f, -4.2725114822387695f, 1.155957579612732f, 7.248950004577637f, 1.4605406522750854f, 2.4984130859375f, -4.926766395568848f, -1.360730767250061f, -3.5312368869781494f, -2.6760661602020264f, 13.249751091003418f, -2.3633241653442383f, 0.16462773084640503f, 2.9756968021392822f, -0.7048916220664978f, 3.0361123085021973f, 2.4167332649230957f, -18.724096298217773f, -2.668437957763672f, 4.492554664611816f, -1.228598952293396f, 4.189570426940918f, 5.219515323638916f, 1.4071485996246338f, -9.033429145812988f, 5.843366622924805f, 9.23009967803955f, -1.3791900873184204f, 2.5881869792938232f, 6.0501861572265625f, 10.04381275177002f, -6.157512187957764f, 7.24165678024292f, -4.1937384605407715f, -8.702505111694336f, 4.425203800201416f, -11.129894256591797f, -3.9875574111938477f, 10.64867877960205f, 5.056559085845947f, -10.747085571289062f, 1.264474630355835f, 7.634984016418457f, -16.58453369140625f, 8.581056594848633f, 6.325976848602295f, 2.503847360610962f, -1.0785939693450928f, 9.530458450317383f, -6.161625862121582f, -7.124795436859131f, 7.468658447265625f, -3.5646467208862305f, -2.393043041229248f, 10.85168743133545f, -1.2992841005325317f, 1.1869677305221558f, 12.415637969970703f, -0.384156733751297f, 1.6292704343795776f, -9.484331130981445f, -1.8839619159698486f, 7.0338287353515625f, -7.619279861450195f, 1.280126690864563f, -3.183976888656616f, -3.0420706272125244f, -3.4128353595733643f, -11.282638549804688f, -2.9777684211730957f, 0.18747295439243317f, -9.763144493103027f, 5.693328857421875f, 6.713829040527344f, 6.432366847991943f, 6.775195121765137f, 15.871235847473145f, -5.289862632751465f, 4.2354960441589355f, -2.9994354248046875f, -1.6106607913970947f, -8.912858009338379f, -9.295467376708984f, -8.503449440002441f, -4.607176303863525f, -1.5842245817184448f, 0.7097844481468201f, 10.443241119384766f, -0.1091175451874733f, -0.7141821384429932f, 0.6914375424385071f, -3.269829750061035f, -8.911444664001465f, -1.1965527534484863f, -1.024635910987854f, 3.5703232288360596f, -1.7879712581634521f, -3.1247658729553223f, -3.874403476715088f, -5.389334678649902f, 15.912910461425781f, -1.5096516609191895f, 10.253766059875488f, 5.733814716339111f, 0.046457380056381226f, -4.8697710037231445f, -6.196042537689209f, 26.0557861328125f, 6.26172399520874f, -6.333352088928223f, 3.3930180072784424f, -10.041016578674316f, -4.201577663421631f, -17.489336013793945f, -7.753904342651367f, -3.3036975860595703f, 9.259572982788086f, 5.622028350830078f, 7.418932914733887f, -4.166983127593994f, 3.4362385272979736f, 7.763801574707031f, 10.344937324523926f, 11.575509071350098f, -6.547769069671631f, 7.228475570678711f, 0.27201998233795166f, 11.231034278869629f, 15.090219497680664f, -3.0461761951446533f, 2.7282047271728516f, 0.22180375456809998f, 0.6711186170578003f, 7.804783821105957f, 1.4017285108566284f, 8.923149108886719f, -12.729116439819336f, 7.182803153991699f, 3.5145866870880127f, -2.1636273860931396f, -22.26779556274414f, -11.131207466125488f, -3.5905392169952393f, 25.306209564208984f, -4.522180557250977f, 8.663426399230957f, -2.190467357635498f, -9.864779472351074f, 0.2012891173362732f, -6.670775890350342f, -4.750314712524414f, 11.050165176391602f, -5.542118549346924f, 9.413408279418945f, 2.308258295059204f, 6.050876617431641f, 5.7319722175598145f, 3.2697877883911133f, -5.117337226867676f, 6.679595947265625f, 3.9800209999084473f, -13.86974048614502f, 4.446991920471191f, -8.73182201385498f, -4.240108966827393f, -3.3107078075408936f, 5.727823734283447f, -1.7871451377868652f, -1.1543947458267212f, 3.274351119995117f, 8.465099334716797f, 4.256385326385498f, 1.6312638521194458f, -0.4266124665737152f, -10.066319465637207f, -1.1442700624465942f, -13.979047775268555f, 9.38019847869873f, 10.103825569152832f, 1.44222891330719f, -12.20266056060791f, -4.161468505859375f, 5.828897476196289f, 3.763794422149658f, 3.17692494392395f, -7.616631507873535f, -4.522240161895752f, 1.3924592733383179f, 3.9834253787994385f, 7.975732326507568f, -5.482377529144287f, -10.70419979095459f, -1.8837058544158936f, -5.173501968383789f, -0.19477716088294983f, -3.4711830615997314f, 10.08415699005127f, -1.9378299713134766f, 5.39734411239624f, -4.766819953918457f, -7.339071750640869f, 4.702202796936035f, 6.930264949798584f, 14.5244779586792f, 1.373946189880371f, -7.17212438583374f, 4.260952949523926f, 13.407155990600586f, 1.6655051708221436f, 9.432223320007324f, 10.111971855163574f, -19.2701473236084f, 7.098327159881592f, -10.078179359436035f, 2.730682373046875f, -6.775233268737793f, 2.490149974822998f, 6.198461532592773f, 0.2401915341615677f, 13.640817642211914f, 6.729548454284668f, -5.323953151702881f, 7.341331958770752f, 0.6824727058410645f, -1.65504789352417f, -13.688631057739258f, 4.6387176513671875f, 3.3633487224578857f, -13.678863525390625f, 5.560359477996826f, -2.417947769165039f, 6.356764316558838f, 11.856796264648438f, -4.349381446838379f, -13.190492630004883f, 0.9404691457748413f, -0.07949790358543396f, 5.1438212394714355f, -5.610644817352295f, 9.887959480285645f, -10.704916000366211f, -6.3003129959106445f, 1.457470178604126f, -5.060678005218506f, 3.0186398029327393f, 0.4794176518917084f, 5.93607759475708f, 5.506703853607178f, -1.0873303413391113f, 2.6558244228363037f, 5.770946502685547f, -4.9762678146362305f, 12.409749031066895f, -7.4365234375f, 8.309518814086914f, 0.3051859140396118f, -6.369776248931885f, -6.938343048095703f, 14.529619216918945f, -8.19320011138916f, 4.215144157409668f, -8.835214614868164f, 2.212118148803711f, 7.021667003631592f, 1.2244818210601807f, -6.6155571937561035f, -5.983504295349121f, 5.949681758880615f, -9.470444679260254f, 1.7039216756820679f, 4.452876567840576f, 3.5223419666290283f, 1.386846661567688f, 12.392443656921387f, 3.4754412174224854f, -3.757160186767578f, 2.243192672729492f, 1.5498485565185547f, -5.969877243041992f, -9.53218936920166f, -0.9648003578186035f, -4.1782612800598145f, -8.63139533996582f, 2.888253688812256f, -4.9197845458984375f, -2.6188461780548096f, -0.34100860357284546f, 1.08323073387146f, 0.20715828239917755f, 0.17625738680362701f, 0.8104458451271057f, 6.235602378845215f, 1.1863518953323364f, 2.2748658657073975f, -2.9206295013427734f, -7.237782001495361f, 7.247722625732422f, -4.8302412033081055f, -16.587038040161133f, 3.3862411975860596f, 5.526927471160889f, -2.761680841445923f, -5.178590774536133f, 12.247193336486816f, -13.684733390808105f, -5.715342998504639f, -8.004195213317871f, 11.39901065826416f, -8.808146476745605f, -11.446958541870117f, -5.106409549713135f, -2.114779472351074f, 0.827870786190033f, 7.99212121963501f, 16.132234573364258f, -7.876288414001465f, 4.045842170715332f, 4.89345121383667f, 7.8902387619018555f, 2.6189887523651123f, 1.4187779426574707f, -4.95357608795166f, 1.9183154106140137f, -10.423624038696289f, 5.698985576629639f, -10.64306640625f, 8.366253852844238f, -6.523881435394287f, -12.701069831848145f, 18.55314064025879f, -17.26905059814453f, -2.0632481575012207f, 8.771729469299316f, 6.320028781890869f, 2.5256295204162598f, 14.239582061767578f, 3.1535520553588867f, 4.2085394859313965f, 0.1063476949930191f, 2.853101968765259f, 7.213879585266113f, 7.257656097412109f, -5.45920467376709f, -4.268126010894775f, -1.759271502494812f, -0.4200115203857422f, 3.3460514545440674f, 3.7233402729034424f, 1.9605106115341187f, 10.866938591003418f, -3.5412983894348145f, 0.06446467339992523f, -13.610902786254883f, -0.3217482566833496f, -13.58945083618164f, 7.48920202255249f, -4.73434591293335f, -4.7902374267578125f, 15.362975120544434f, -2.182408571243286f, 5.474664211273193f, -9.107425689697266f, 1.9057763814926147f, 5.691718101501465f, -5.837611675262451f, 0.9479477405548096f, 5.760697841644287f, 0.09852147847414017f, 4.3325629234313965f, -1.1939945220947266f, -9.364533424377441f, 5.033231735229492f, 4.058061122894287f, -13.785080909729004f, 1.9617301225662231f, -15.00662899017334f, 1.1908645629882812f, 1.5933691263198853f, -3.296618938446045f, 0.3798801302909851f, 3.9930577278137207f, 13.144612312316895f, -19.245630264282227f, 3.366441011428833f, -9.155226707458496f, -4.203514099121094f, -6.379355430603027f, 3.052100896835327f, 3.3825178146362305f, -5.03127384185791f, -20.389877319335938f, 15.295291900634766f, -17.634958267211914f, 7.198425769805908f, -26.755468368530273f, -5.668976783752441f, 9.652477264404297f, 1.5629017353057861f, -3.2165329456329346f, -2.528200626373291f, 8.87691593170166f, 1.8550244569778442f, 12.378725051879883f, 0.00554652651771903f, 5.4776787757873535f, -0.582221508026123f, -3.155843734741211f, 12.177905082702637f, -0.9750184416770935f, -0.16764011979103088f, -19.229124069213867f, 5.547354221343994f, 14.744590759277344f, 7.076948642730713f, 11.192045211791992f, 8.736725807189941f, -10.47435188293457f, -1.8428961038589478f, 8.822990417480469f, 10.079815864562988f, -9.724599838256836f, 4.329649925231934f, 22.17165756225586f, -12.209203720092773f, 6.37702751159668f, -15.575246810913086f, 4.703880786895752f, -1.969223976135254f, 0.45949697494506836f, 0.20915208756923676f, 11.847763061523438f, -3.744934558868408f, -13.487935066223145f, 3.874084234237671f, -4.578466415405273f, -7.086031436920166f, -6.233429908752441f, -6.785592555999756f, -3.3219637870788574f, 8.725717544555664f, 8.299102783203125f, 12.64601993560791f, -5.9381513595581055f, 0.5933879613876343f, 10.182705879211426f, 5.442933559417725f, 1.8527637720108032f, -4.759885787963867f, -4.977668285369873f, -7.128690719604492f, -2.2508225440979004f, -5.206679344177246f, 1.6984038352966309f, 3.4979476928710938f, -0.9469919204711914f, -5.347060680389404f, 2.9652979373931885f, -3.192228317260742f, 4.303493499755859f, 9.290297508239746f, 9.933444023132324f, -5.329427719116211f, 2.070354700088501f, 2.8292171955108643f, 13.59816837310791f, -2.8414688110351562f, 2.1585311889648438f, 5.236742973327637f, -2.824960947036743f, -18.918386459350586f, 0.8947755098342896f, -7.695527076721191f, 1.7885832786560059f, -18.260061264038086f, -15.04204273223877f, -1.8974013328552246f, -9.819128036499023f, 8.226827621459961f, -5.237205505371094f, 4.667030334472656f, -13.77105712890625f, 4.586790084838867f, -0.390918493270874f, -4.865528106689453f, -5.15276575088501f, 8.497437477111816f, 1.938669204711914f, -4.695268154144287f, 8.078354835510254f, 4.665389537811279f, -8.80441665649414f, 10.505725860595703f, -7.680267810821533f, -1.3688803911209106f, -3.3594307899475098f, 5.555975437164307f, 11.535223007202148f, -0.9209914803504944f, 2.747661590576172f, -14.198168754577637f, -2.046473503112793f, 9.80955982208252f, 0.09400451183319092f, 7.27423095703125f, 8.030008316040039f, -7.390738487243652f, 0.04207156226038933f, 6.497010231018066f, 2.9516983032226562f, -5.786115646362305f, 6.287818908691406f, 14.243786811828613f, -8.465189933776855f, -1.7876099348068237f, -5.9232072830200195f, 5.831533908843994f, -6.165143013000488f, 6.243556499481201f, -3.9629876613616943f, 10.030241966247559f, 13.016631126403809f, -6.402768611907959f, -3.428553819656372f, -2.3323075771331787f, 0.22513322532176971f, 9.516219139099121f, 8.901734352111816f, 1.7534918785095215f, 6.5073628425598145f, -11.850318908691406f, 22.981639862060547f, 0.9134898781776428f, -6.708924293518066f, 8.352168083190918f, 6.139276027679443f, -11.110394477844238f, -12.100716590881348f, 3.7225136756896973f, -6.486250400543213f, 4.79709529876709f, 3.2657382488250732f, -6.960175514221191f, 4.827823162078857f, 3.0818111896514893f, 4.233963966369629f, -1.300526738166809f, 8.41450023651123f, 27.55376625061035f, 6.323548793792725f, 1.2947889566421509f, -4.953489303588867f, 1.9106656312942505f, 1.7545992136001587f, 4.909358024597168f, -2.6168389320373535f, -13.989197731018066f, 2.96791672706604f, -15.628793716430664f, -4.646886348724365f, 2.3803651332855225f, -0.38021320104599f, 7.955515384674072f, 10.828896522521973f, -4.704763412475586f, -2.4767675399780273f, 4.345104694366455f, 9.674301147460938f, 14.051787376403809f, -1.9404326677322388f, -6.1034440994262695f, 4.96730375289917f, 8.356350898742676f, -12.492217063903809f, 5.052522659301758f, -11.288028717041016f, -18.85377311706543f, -2.1773197650909424f, 3.268491268157959f, -11.059211730957031f, 12.636406898498535f, 8.842144966125488f, 4.771931171417236f, 13.92164134979248f, -9.583444595336914f, 18.409446716308594f, -7.4875359535217285f, 24.48175621032715f, 5.571856498718262f, 3.763981819152832f, 2.6589486598968506f, -6.752713680267334f, -14.821791648864746f, -4.6671833992004395f, -7.933231830596924f, -5.372705459594727f, -16.37500762939453f, 8.258382797241211f, 4.597978115081787f, -4.5285491943359375f, -16.733186721801758f, -15.706796646118164f, -9.255148887634277f, 5.370093822479248f, -2.010138988494873f, 3.0411553382873535f, 5.09183406829834f, -0.09162905812263489f, 5.09684419631958f, 7.237029075622559f, -1.3704605102539062f, 15.3106107711792f, -9.188556671142578f, -2.035550117492676f, -11.181009292602539f, -3.858069658279419f, 0.42419710755348206f, 10.823795318603516f, 2.402636766433716f, -2.0442917346954346f, 2.90970778465271f, -10.479278564453125f, -5.484250068664551f, -5.404007434844971f, 5.897701263427734f, -10.198234558105469f, 3.919404983520508f, 11.954742431640625f, -3.368873119354248f, -2.416825532913208f, -5.485574245452881f, -1.0831294059753418f, 12.187384605407715f, 9.810623168945312f, 2.734238624572754f, -3.1333184242248535f, -5.630797386169434f, -3.6640121936798096f, -5.316211223602295f, -5.74959135055542f, -0.2944042682647705f, -23.43301010131836f, -3.242238998413086f, 3.7050588130950928f, 10.571874618530273f, 10.871262550354004f, 0.4957568347454071f, -13.565269470214844f, -0.34055426716804504f, -5.27890682220459f, -8.633820533752441f, -4.941458702087402f, -1.589221715927124f, 0.5624170899391174f, 5.252312183380127f, -6.480530261993408f, 1.477312445640564f, 3.0334291458129883f, 1.7544362545013428f, -1.811592698097229f, 2.166680097579956f, -2.746460199356079f, -14.217162132263184f, 0.9965202212333679f, -3.636343479156494f, -4.826642990112305f, -3.4294586181640625f, -13.18789291381836f, 1.1160309314727783f, 0.03090650774538517f, 4.610648155212402f, -6.316239356994629f, 5.343184947967529f, -3.5397722721099854f, 13.128379821777344f, 6.051750659942627f, 3.017526626586914f, -0.22869278490543365f, 1.0593830347061157f, -4.756152153015137f, -9.622599601745605f, -14.170649528503418f, 2.4119977951049805f, 3.543015241622925f, -6.777645111083984f, -2.0697014331817627f, -1.148790717124939f, -5.165157318115234f, -2.193115472793579f, 2.605295181274414f, -12.414362907409668f, -16.186542510986328f, -5.055665969848633f, 14.077547073364258f, -9.300932884216309f, 4.403715133666992f, -3.700418710708618f, 1.266947865486145f, 4.215131759643555f, 10.175348281860352f, -0.7335017919540405f, -6.666941165924072f, 1.7698924541473389f, 2.5551509857177734f, 14.33661937713623f, -6.243782043457031f, -7.089589595794678f, 5.264889240264893f, 2.0064051151275635f, -1.1940221786499023f, -0.8845052719116211f, 4.728024959564209f, 9.143659591674805f, -11.243470191955566f, -3.881427764892578f, 1.0975744724273682f, 5.020593166351318f, -2.509296178817749f, -0.7040889263153076f, -8.826937675476074f, 9.25462818145752f, 0.6523112058639526f, 8.413361549377441f, -9.956643104553223f, 4.320390701293945f, -7.327249050140381f, 0.7185436487197876f, 5.529834747314453f, 3.7425594329833984f, 11.977640151977539f, 0.3199857473373413f, -1.2970579862594604f, -0.6297903656959534f, -6.748528480529785f, -1.8058745861053467f, 7.02766227722168f, -14.052090644836426f, 3.328584909439087f, -11.310714721679688f, -8.255908966064453f, -1.6174887418746948f, 2.040614604949951f, -7.178774833679199f, -8.185322761535645f, -12.005358695983887f, -1.7305762767791748f, -15.079358100891113f, -8.002707481384277f, -13.309523582458496f, -4.7150774002075195f, -7.385843276977539f, 3.19026517868042f, -0.39262112975120544f, 10.768528938293457f, -12.257975578308105f, -20.101083755493164f, -10.958288192749023f, 7.338348388671875f, -5.042275905609131f, -4.23043966293335f, -4.301355361938477f, 6.800642967224121f, 0.3120475709438324f, 10.304023742675781f, -13.633195877075195f, -0.5820647478103638f, 5.642172813415527f, -7.806666851043701f, 4.420014381408691f, -11.09785270690918f, -0.6035222411155701f, -3.352290630340576f, -11.287544250488281f, -9.170281410217285f, 4.307720184326172f, 5.357173442840576f, 9.775383949279785f, -4.904134273529053f, 0.8738774657249451f, -1.9923650026321411f, -13.437588691711426f, 4.374331951141357f, -6.224425315856934f, -7.205249786376953f, -1.2282158136367798f, 16.407073974609375f, 14.745433807373047f, 10.053177833557129f, -5.026731014251709f, 13.399908065795898f, -4.700345039367676f, -2.810746908187866f, 4.868433952331543f, 14.18775463104248f, 0.07687458395957947f, -1.2327467203140259f, -1.3728224039077759f, 7.709836483001709f, 3.666210889816284f, 4.985335826873779f, -12.233044624328613f, -5.2307329177856445f, -5.411324977874756f, 5.240880012512207f, -2.543198347091675f, -21.135807037353516f, -11.415369987487793f, 5.937788486480713f, -0.01360476203262806f, -15.29384708404541f, -23.956432342529297f, 1.6906027793884277f, 3.3760499954223633f, 9.199305534362793f, 12.791433334350586f, 4.497912883758545f, 3.4790523052215576f, -2.8718466758728027f, 4.7231268882751465f, -12.663939476013184f, 8.505736351013184f, 0.8276928663253784f, -6.634997367858887f, 3.213438034057617f, -12.446330070495605f, -0.040080517530441284f, 8.142585754394531f, -5.221867084503174f, -7.002190113067627f, -0.7204623818397522f, -11.662397384643555f, -1.66923189163208f, 2.7600550651550293f, -7.987179279327393f, 1.4977517127990723f, -1.5806363821029663f, -3.7816100120544434f, 0.5635890960693359f, 1.5413968563079834f, -1.8732370138168335f, 0.9285376667976379f, -21.148822784423828f, 19.22353744506836f, -2.7432806491851807f, -14.096722602844238f, 8.600056648254395f, -8.63492488861084f, -0.5446416139602661f, -9.644774436950684f, 1.5598598718643188f, 9.11766242980957f, 11.372527122497559f, -12.296475410461426f, 7.532688617706299f, -8.911373138427734f, 6.434876441955566f, 3.617435932159424f, 4.387059688568115f, 12.811718940734863f, -9.318404197692871f, -0.4164440929889679f, -4.88645601272583f, -9.689740180969238f, -7.375895023345947f, 3.4132399559020996f, -5.05049991607666f, -4.028200626373291f, -8.181990623474121f, -7.417090892791748f, -6.058056831359863f, 4.370342254638672f, 4.167178630828857f, 10.502394676208496f, -8.585572242736816f, -7.519354820251465f, 11.942482948303223f, 13.272440910339355f, -2.757564067840576f, 1.3308485746383667f, -2.266592264175415f, -6.452311992645264f, -8.549099922180176f, -6.8195576667785645f, -6.657449722290039f, -11.0411376953125f, -21.244892120361328f, -9.757211685180664f, 3.444068670272827f, 5.706812858581543f, -1.7444701194763184f, 8.521875381469727f, -5.228854179382324f, -5.382219314575195f, -6.763889789581299f, -11.332088470458984f, -5.202573299407959f, 1.0299209356307983f, -11.423958778381348f, -1.0182582139968872f, 12.390052795410156f, -10.38305950164795f, -5.015763282775879f, 10.922207832336426f, -2.7742760181427f, 16.766313552856445f, -7.944525241851807f, -2.8002591133117676f, 10.874272346496582f, -9.857773780822754f, 2.1176915168762207f, -7.479413032531738f, -1.7775098085403442f, 8.738555908203125f, -1.012858271598816f, -0.524698793888092f, -2.0892815589904785f, 2.1344618797302246f, 2.1381497383117676f, -8.60706615447998f, -3.834965944290161f, 5.380415916442871f, -5.791232109069824f, 12.458983421325684f, -2.1414995193481445f, -6.775130271911621f, 6.628716468811035f, -6.176581382751465f, 1.7703351974487305f, 4.631689071655273f, 6.3223066329956055f, 2.441192388534546f, -5.126554012298584f, -1.0228726863861084f, 2.649749517440796f, 2.357867956161499f, -2.623882532119751f, 0.692119300365448f, 7.061679840087891f, -1.3346446752548218f, 9.300801277160645f, -2.3639140129089355f, -14.160957336425781f, 4.440097808837891f, 1.5377870798110962f, -2.905083179473877f, -14.448901176452637f, 3.674952983856201f, -3.8605237007141113f, 8.271306037902832f, 6.405794620513916f, -21.742176055908203f, -11.455822944641113f, 1.7873941659927368f, -11.738869667053223f, 8.434774398803711f, -20.15655517578125f, -1.367598056793213f, 7.51062536239624f, -7.572475910186768f, -0.1449132263660431f, 0.14903344213962555f, 1.539661169052124f, -8.20009994506836f, -7.516574382781982f, 6.837973117828369f, -5.781085968017578f, 1.294516921043396f, -9.226051330566406f, 1.2180476188659668f, -4.204068183898926f, 15.320548057556152f, -6.9747209548950195f, 10.799696922302246f, 9.24210262298584f, 9.05672550201416f, -7.968204498291016f, -4.376702308654785f, -2.1221911907196045f, 0.1324523389339447f, 4.775428771972656f, -0.22519434988498688f, -7.35526180267334f, 0.1219637468457222f, -11.087621688842773f, -7.007081985473633f, 5.306663990020752f, -16.567237854003906f, -9.150774002075195f, 19.48033332824707f, 9.253161430358887f, 8.751167297363281f, 6.719225883483887f, 5.08009672164917f, -6.268258571624756f, 0.8974515795707703f, -12.448920249938965f, 19.20334815979004f, -12.337990760803223f, -2.6060383319854736f, 17.232389450073242f, -5.7053303718566895f, -2.461069107055664f, -9.090248107910156f, -2.2077951431274414f, 5.545995235443115f, 2.147212028503418f, 8.779653549194336f, -1.8891314268112183f, 10.41473388671875f, 5.098225116729736f, 3.8328142166137695f, 4.6025872230529785f, -3.5405564308166504f, 4.929543972015381f, -15.190320014953613f, -4.475381851196289f, 4.857414722442627f, -0.2330942153930664f, -9.112611770629883f, -14.155166625976562f, -14.117636680603027f, 0.3170005977153778f, -6.148338794708252f, 2.6847426891326904f, 1.3375723361968994f, -10.719864845275879f, 19.82689666748047f, -11.773765563964844f, 1.4471533298492432f, 5.438664436340332f, 10.833401679992676f, 5.91309928894043f, -4.8504557609558105f, -0.8866681456565857f, 4.502740383148193f, 8.76336669921875f, -7.2705078125f, -2.602914810180664f, 0.4528222978115082f, 0.2632019817829132f, -3.9463186264038086f, -6.615810394287109f, -12.216948509216309f, 18.130096435546875f, -11.216374397277832f, -9.20461368560791f, 2.837886333465576f, -9.17361068725586f, -13.872835159301758f, 1.4739644527435303f, -4.641720294952393f, -6.107040882110596f, -0.7774397730827332f, 2.1887550354003906f, -3.053284168243408f, 6.492974758148193f, -8.238874435424805f, -17.8471622467041f, -5.672789096832275f, -2.3216474056243896f, -1.9701411724090576f, -7.030838966369629f, -2.107118844985962f, -0.4771590828895569f, -1.8420956134796143f, -14.038286209106445f, -4.026478290557861f, 3.295030355453491f, 4.460195064544678f, 6.017807960510254f, 7.253128528594971f, -7.15041971206665f, 0.515494167804718f, 4.030392646789551f, -3.3914072513580322f, 0.44524064660072327f, -0.19258594512939453f, -0.7080201506614685f, -2.50250244140625f, 2.334340810775757f, -11.007967948913574f, -12.750214576721191f, -7.300755977630615f, 14.395295143127441f, -3.3914880752563477f, 8.228107452392578f, 9.74307918548584f, 0.77601557970047f, 2.347871780395508f, 9.640531539916992f, -4.919435024261475f, -7.991837024688721f, 0.03702786564826965f, 0.6593180894851685f, -8.348962783813477f, 4.535181999206543f, 3.7413909435272217f, -4.0721588134765625f, 7.057863712310791f, -1.0119569301605225f, -5.5510711669921875f, -11.117044448852539f, 5.491680145263672f, 12.420897483825684f, 12.342089653015137f, -13.135498046875f, 0.531191349029541f, -3.1611971855163574f, 2.2477285861968994f, -4.986516952514648f, 7.761373996734619f, 2.0133655071258545f, 9.318339347839355f, 8.271308898925781f, 2.2258410453796387f, 7.38371467590332f, 1.7697837352752686f, 7.604837894439697f, -5.065467357635498f, 15.182759284973145f, 10.54971694946289f, 14.205101013183594f, -15.358622550964355f, -7.208074569702148f, -1.3423936367034912f, 4.829193115234375f, -6.763162612915039f, -6.093154430389404f, -3.8595669269561768f, 10.870450019836426f, 5.670471668243408f, -9.437480926513672f, 2.4107041358947754f, -6.857131004333496f, 2.4801251888275146f, -5.085203170776367f, 15.741228103637695f, -18.639524459838867f, 0.10690624266862869f, 10.266143798828125f, 0.7210645079612732f, 2.90657114982605f, 4.488924980163574f, 11.784412384033203f, 10.836816787719727f, 11.812047004699707f, 0.9760425090789795f, 6.398825645446777f, -11.69657039642334f, 9.617613792419434f, -2.4754092693328857f, 2.3830978870391846f, 4.122246742248535f, -6.237287521362305f, 6.2666802406311035f, -7.420814514160156f, -4.996058940887451f, 6.019990921020508f, 8.033538818359375f, -3.3236873149871826f, -10.9536714553833f, 6.0252366065979f, -10.10529613494873f, 10.607632637023926f, -1.0954471826553345f, -7.890513896942139f, -6.947789669036865f, 18.596343994140625f, -10.062642097473145f, 9.224631309509277f, 12.48892593383789f, 11.919086456298828f, 10.027180671691895f, 8.543699264526367f, -8.874430656433105f, -2.0335042476654053f, -14.272953033447266f, 3.9183735847473145f, -8.344069480895996f, -11.0534086227417f, 2.4750025272369385f, -0.029886547476053238f, 5.07313871383667f, -26.66839599609375f, -8.733036041259766f, -4.485158920288086f, 9.778528213500977f, 5.770241737365723f, 12.092632293701172f, 1.5609368085861206f, 11.50508975982666f, -0.6498426795005798f, -9.589741706848145f, -27.592344284057617f, -0.43285009264945984f, 11.695541381835938f, -6.151013374328613f, 5.020718097686768f, -12.249852180480957f, -2.1764540672302246f, 6.811885833740234f, 16.927019119262695f, -10.076769828796387f, -17.306066513061523f, 4.369997978210449f, 1.4093303680419922f, -2.871824026107788f, 2.7676126956939697f, -6.874851226806641f, -2.597337484359741f, -9.963288307189941f, 10.695664405822754f, 5.037891387939453f, -6.623497009277344f, 5.716047286987305f, 4.312154769897461f, 4.035192489624023f, 2.1116819381713867f, 3.357062578201294f, 1.416142225265503f, 3.6634714603424072f, -4.913036823272705f, -5.307648181915283f, 11.786884307861328f, -1.3370147943496704f, 10.858242988586426f, 8.861958503723145f, 6.27660608291626f, 12.768265724182129f, 12.327997207641602f, -8.675048828125f, -3.4217770099639893f, 7.692430019378662f, 1.7838584184646606f, 12.854020118713379f, -11.746981620788574f, -9.584903717041016f, 8.365598678588867f, 12.973992347717285f, 5.4280571937561035f, -9.301798820495605f, 11.192365646362305f, -2.602813243865967f, 6.336965084075928f, 0.5768340229988098f, -10.169051170349121f, -8.006020545959473f, 13.611660957336426f, -5.536235332489014f, 3.161245346069336f, -9.059117317199707f, -2.214181423187256f, -5.4537811279296875f, 6.635800838470459f, -12.098894119262695f, 11.331655502319336f, 13.68090534210205f, 3.1792664527893066f, 1.3622552156448364f, 2.2142395973205566f, 0.2345779687166214f, 0.5814613103866577f, 4.289124011993408f, 4.538444995880127f, -7.362942695617676f, 2.874087333679199f, 3.61981201171875f, -5.420032501220703f, 8.720592498779297f, 4.290198802947998f, 12.945777893066406f, 5.480059623718262f, -7.588292598724365f, 5.661287784576416f, -3.8283472061157227f, -8.462838172912598f, 7.285812854766846f, -10.747210502624512f, -3.003866672515869f, 7.091235637664795f, -1.580762505531311f, 8.499457359313965f, 0.9311726689338684f, 2.294097661972046f, 7.290738582611084f, -4.4237260818481445f, 13.710098266601562f, -7.216895580291748f, 2.5698392391204834f, -4.586067199707031f, -5.298479080200195f, -3.6828322410583496f, -1.1227331161499023f, 6.324790954589844f, 0.32012125849723816f, -0.4937366843223572f, -1.9536423683166504f, -9.237577438354492f, 1.0877151489257812f, 12.129581451416016f, 4.084137916564941f, -2.9383089542388916f, -5.019631862640381f, 0.6925498247146606f, 6.584831714630127f, 0.4018475115299225f, 0.21690116822719574f, 1.1919173002243042f, 2.2553939819335938f, -1.0278080701828003f, -13.67727279663086f, -1.5738617181777954f, 2.352642297744751f, 3.2829935550689697f, 0.22515998780727386f, -5.764885902404785f, -3.821402072906494f, 7.599716663360596f, 4.4758172035217285f, -18.555747985839844f, 10.015554428100586f, -5.277355194091797f, 4.857947826385498f, -4.972664833068848f, -7.69036865234375f, 0.8392868041992188f, -26.074081420898438f, -8.01816463470459f, 17.92840576171875f, 4.751870632171631f, 1.765329360961914f, -4.687397003173828f, 8.163311958312988f, 4.182957172393799f, 8.034849166870117f, 1.3160547018051147f, -0.5854548215866089f, -8.705289840698242f, -0.8203011155128479f, -3.8902640342712402f, -3.927004814147949f, 10.755240440368652f, -2.724745035171509f, -5.045247554779053f, -13.067852973937988f, 15.737813949584961f, 1.6259382963180542f, 0.018999869003891945f, 1.839993953704834f, 2.3978917598724365f, 1.3756414651870728f, 10.791954040527344f, -4.205836772918701f, 7.3124308586120605f, 3.0303399562835693f, 18.43343734741211f, 2.922717571258545f, 7.7857747077941895f, 9.202900886535645f, -11.383536338806152f, 4.063195705413818f, 4.0933308601379395f, 9.472505569458008f, 0.6550918817520142f, 1.219048261642456f, -3.383958101272583f, -13.222259521484375f, 6.909287452697754f, -11.573260307312012f, 4.1832146644592285f, 6.807440280914307f, -12.13596248626709f, -16.246288299560547f, -11.497807502746582f, 8.08572769165039f, -12.573715209960938f, -4.854974746704102f, -2.2262701988220215f, 0.8039400577545166f, 11.886690139770508f, -15.268168449401855f, -8.579140663146973f, -2.6850054264068604f, 5.244555950164795f, -6.70413064956665f, -3.2002172470092773f, 6.914629936218262f, 0.3705389201641083f, 14.895472526550293f, -6.253382205963135f, -6.998981475830078f, 4.563458442687988f, 5.477730751037598f, -9.057239532470703f, 9.500064849853516f, 4.373735427856445f, -0.7703867554664612f, 13.951043128967285f, -5.243332862854004f, 2.8435473442077637f, -6.590154647827148f, 4.319807529449463f, -2.5442638397216797f, -13.913573265075684f, -10.475417137145996f, -2.8347291946411133f, 8.493965148925781f, -10.500249862670898f, -5.360538005828857f, -4.234364986419678f, 0.10634779930114746f, -1.185592532157898f, -9.338807106018066f, 2.1675655841827393f, 3.200042486190796f, 4.2734856605529785f, 5.105844020843506f, -7.451834201812744f, -11.285500526428223f, -2.5912656784057617f, -4.90095853805542f, -7.117650508880615f, 5.817300796508789f, -2.694540023803711f, -6.723092555999756f, 3.3572030067443848f, 9.950608253479004f, -2.062587261199951f, -2.237545967102051f, 5.24613094329834f, 18.30078887939453f, -7.8479437828063965f, -2.280181407928467f, 8.82515811920166f, 0.1949726939201355f, 4.303898811340332f, -3.4921815395355225f, 16.898700714111328f, 11.57261848449707f, -2.4016921520233154f, -3.2015914916992188f, 10.793314933776855f, -6.424396514892578f, 1.5789941549301147f, 6.381648540496826f, -21.38960838317871f, 0.6480197906494141f, -13.997282028198242f, 0.6987282037734985f, -7.389518737792969f, 10.640852928161621f, -1.155595064163208f, -2.187462091445923f, -2.0233631134033203f, -2.092860698699951f, 8.194184303283691f, 12.357358932495117f, 12.201729774475098f, 5.951974868774414f, 1.862755537033081f, -4.316686630249023f, 6.706587791442871f, 5.658695220947266f, -3.7403807640075684f, 4.848812580108643f, -1.1044867038726807f, 4.073699474334717f, -6.804827690124512f, -2.227973461151123f, -14.886821746826172f, 4.097927093505859f, -5.149636745452881f, 14.20467472076416f, 10.393085479736328f, 16.598787307739258f, 20.412975311279297f, 0.4636867642402649f, -12.83670711517334f, 5.011075496673584f, 3.2573134899139404f, 2.541311502456665f, -3.704630136489868f, -7.621809482574463f, 2.2266361713409424f, -6.034920692443848f, -4.907690525054932f, 3.311673164367676f, -11.02491569519043f, 6.5720438957214355f, -13.933245658874512f, 3.847233295440674f, 12.274158477783203f, 0.9272652864456177f, -1.7610269784927368f, 0.4040965139865875f, 0.680636465549469f, -15.860685348510742f, 1.9717633724212646f, -9.28317928314209f, -2.6344847679138184f, 1.870112657546997f, -15.349178314208984f, -5.096085548400879f, -1.4747427701950073f, -8.14248275756836f, 2.2569987773895264f, -15.152320861816406f, -7.701322078704834f, 10.674273490905762f, -1.4894379377365112f, -4.1722493171691895f, -20.385786056518555f, -7.458427429199219f, 0.15474386513233185f, 6.180734634399414f, 9.458477973937988f, 3.9757118225097656f, 3.1847710609436035f, 2.5685551166534424f, -1.0029635429382324f, -0.9310358762741089f, -5.389017581939697f, 13.258286476135254f, 4.855055809020996f, 11.788922309875488f, 2.5168514251708984f, -8.065077781677246f, 5.255640983581543f, 12.24008846282959f, -13.184330940246582f, -0.7134942412376404f, -8.317489624023438f, 11.638908386230469f, -3.975881814956665f, 0.43127647042274475f, -8.266470909118652f, -12.825586318969727f, 13.573546409606934f, 8.546313285827637f, 4.274265289306641f, 3.6718971729278564f, 10.517722129821777f, 0.27425989508628845f, 8.075490951538086f, -2.1264874935150146f, 0.6136637926101685f, -5.506547451019287f, 15.231038093566895f, 0.46134939789772034f, 0.17480090260505676f, 2.040527582168579f, 3.2183902263641357f, -6.139379501342773f, -9.661615371704102f, -10.546405792236328f, -3.0357425212860107f, -3.1846868991851807f, -4.761189937591553f, -3.7215702533721924f, 3.0658512115478516f, -11.569316864013672f, -0.43601199984550476f, 2.8654866218566895f, -2.9958226680755615f, 2.4096181392669678f, -10.564579963684082f, -13.85832405090332f, -6.305012226104736f, 7.627152919769287f, 2.5408518314361572f, -10.277200698852539f, 1.9411969184875488f, -4.243484973907471f, 1.4209707975387573f, 3.253671169281006f, 1.769997000694275f, -9.947007179260254f, 3.4785919189453125f, 3.4994893074035645f, 14.112330436706543f, 17.531051635742188f, 3.3806848526000977f, 2.1774063110351562f, -14.381307601928711f, 6.56845235824585f, -11.52322769165039f, 0.6346222162246704f, 7.065883636474609f, -3.087952136993408f, 6.3684282302856445f, -6.367730140686035f, 6.013090133666992f, 4.485490322113037f, 3.8508636951446533f, -11.656087875366211f, 0.451434463262558f, 7.0519280433654785f, 6.291365623474121f, 10.597298622131348f, 7.171876430511475f, 1.4291812181472778f, -8.897068977355957f, 0.9057914614677429f, 1.817928671836853f, -3.542912006378174f, 15.661563873291016f, -3.5200037956237793f, -1.275672435760498f, 11.505273818969727f, -5.713345527648926f, -13.151139259338379f, 1.984048843383789f, 24.558940887451172f, -0.4125036895275116f, 7.0134735107421875f, 14.921035766601562f, 3.6945953369140625f, 15.289679527282715f, -12.831208229064941f, -7.242607593536377f, -14.355047225952148f, -3.520359992980957f, 17.282093048095703f, -1.093096137046814f, -4.698165416717529f, 4.53480863571167f, -2.2949156761169434f, 2.932422399520874f, 18.38007926940918f, 10.144601821899414f, 1.8887478113174438f, 17.05137825012207f, 2.2348875999450684f, 2.445681571960449f, -16.11703109741211f, 3.6988015174865723f, 28.14395523071289f, -9.436356544494629f, -4.76363468170166f, -9.584181785583496f, -0.4704954922199249f, 1.5917537212371826f, -21.702960968017578f, 6.3507513999938965f, -3.284785270690918f, 7.535861968994141f, 11.226126670837402f, 9.685650825500488f, 0.8183719515800476f, -15.07618236541748f, 6.108888149261475f, 16.537776947021484f, 12.913955688476562f, -4.378338813781738f, 7.780423641204834f, 7.1294660568237305f, 1.4495526552200317f, 0.26958322525024414f, 4.329139709472656f, 6.674724102020264f, -6.520862579345703f, -2.903130531311035f, -17.643451690673828f, 7.825291633605957f, -4.506794452667236f, 0.16557633876800537f, 7.2597246170043945f, 8.140376091003418f, -9.9315185546875f, -6.21274471282959f, -3.006944179534912f, 4.115133285522461f, 1.6233476400375366f, -8.518281936645508f, 1.5647332668304443f, -7.484006881713867f, -4.436017036437988f, 9.502815246582031f, -7.330345630645752f, 7.664921760559082f, -4.133516788482666f, 3.0703797340393066f, 7.408110618591309f, -1.3500370979309082f, -3.113905191421509f, -1.6690822839736938f, 9.191621780395508f, 0.6589356064796448f, -7.368348121643066f, 11.585264205932617f, 2.6871490478515625f, 5.345765113830566f, 1.0851669311523438f, -4.77912712097168f, -9.00320816040039f, -8.105009078979492f, 3.760385274887085f, 8.186501502990723f, 13.72744369506836f, 4.232875823974609f, 4.095627307891846f, 1.6133935451507568f, 14.020605087280273f, -17.89886474609375f, 10.849470138549805f, -4.106210708618164f, -1.5198999643325806f, -4.078025817871094f, 3.1078972816467285f, 6.467252254486084f, 8.850523948669434f, -1.770440936088562f, -3.601609706878662f, 12.48472785949707f, 1.1591122150421143f, 3.630988597869873f, 0.35214078426361084f, 9.08293628692627f, -9.61496353149414f, -3.6545019149780273f, 4.853860378265381f, -21.803064346313477f, 0.9578412175178528f, -5.081964492797852f, -3.8409125804901123f, 13.125861167907715f, 8.514156341552734f, 2.2802765369415283f, -1.5544071197509766f, 7.989774227142334f, -0.9534971117973328f, -20.655189514160156f, 5.728123664855957f, -4.578756809234619f, -9.761033058166504f, -2.411184072494507f, -1.53468918800354f, 1.7299505472183228f, 4.49857759475708f, -5.621171951293945f, -15.699580192565918f, -1.6200847625732422f, 0.2511891722679138f, 12.249534606933594f, -6.566256523132324f, -14.719828605651855f, -11.666041374206543f, 1.057192087173462f, -1.8811215162277222f, -4.319576263427734f, 9.619815826416016f, -6.46962833404541f, -3.5396673679351807f, -4.881474494934082f, -0.6955579519271851f, 3.299593687057495f, 11.470381736755371f, 1.12203049659729f, -0.921830952167511f, 5.743063449859619f, 6.646578311920166f, 1.6719251871109009f, -11.525588989257812f, -12.89569091796875f, 7.118905544281006f, -14.61849594116211f, -9.530622482299805f, -6.113729476928711f, 0.5888143181800842f, -17.381607055664062f, 15.451555252075195f, -6.282611846923828f, -12.82925033569336f, 1.7211908102035522f, -12.8479585647583f, 3.2159461975097656f, 2.13104510307312f, 0.8415284752845764f, -4.35218620300293f, -6.643268585205078f, -3.2159981727600098f, -1.1216778755187988f, -9.839357376098633f, -6.70488977432251f, -4.389339447021484f, -3.2769744396209717f, -5.768002510070801f, -8.732684135437012f, -5.264852046966553f, 6.483272552490234f, 9.102012634277344f, -14.003499984741211f, 0.17809408903121948f, 15.125717163085938f, -12.025708198547363f, -6.3086838722229f, 14.6791353225708f, 9.44178581237793f, -8.969319343566895f, -5.2899250984191895f, -9.268863677978516f, -6.997377872467041f, 9.275823593139648f, 1.1657955646514893f, -7.220368385314941f, 10.789793968200684f, 10.304367065429688f, -4.045272350311279f, -6.108043193817139f, -2.014784812927246f, -11.121515274047852f, -18.38974380493164f, -8.347667694091797f, 5.974706649780273f, 10.243739128112793f, 5.663839340209961f, 8.547972679138184f, -7.52782678604126f, 5.363646507263184f, 15.58993148803711f, 7.026348114013672f, 2.677098512649536f, 18.409914016723633f, -10.548135757446289f, 3.4355015754699707f, 1.4359816312789917f, -2.546203374862671f, -6.266178131103516f, 11.344420433044434f, -19.2717227935791f, -6.462072372436523f, 11.376580238342285f, 3.8701956272125244f, -8.504956245422363f, -7.087599277496338f, -11.178908348083496f, 0.2572647035121918f, 4.125010967254639f, 9.298619270324707f, -8.014323234558105f, 12.535018920898438f, 6.390746116638184f, -6.37876558303833f, 6.679197788238525f, -9.323160171508789f, -5.269445419311523f, 2.596299886703491f, 0.11649034917354584f, 17.804271697998047f, 4.594757556915283f, 6.764013767242432f, 0.6217585206031799f, -7.7316412925720215f, 8.208429336547852f, -7.3180928230285645f, -9.159605979919434f, 8.136429786682129f, -15.975767135620117f, -5.516753673553467f, -7.645376205444336f, -6.816018104553223f, -1.813738226890564f, -0.754987359046936f, 10.540684700012207f, 6.572773456573486f, 4.151686191558838f, 5.905804634094238f, -0.29068470001220703f, -5.357999324798584f, -5.847013473510742f, -4.441774845123291f, 1.8924025297164917f, -1.1913574934005737f, 14.646678924560547f, -13.63921070098877f, 1.4032258987426758f, -2.480900526046753f, 0.2637772262096405f, -7.374143600463867f, 5.216763019561768f, -22.21915626525879f, -1.6103098392486572f, 9.66075325012207f, 7.245777606964111f, -0.4308447539806366f, 3.232591152191162f, 7.705619812011719f, -3.5778160095214844f, -4.7608866691589355f, -3.9943084716796875f, -2.1508443355560303f, 7.583977222442627f, -11.541088104248047f, -9.87782096862793f, -0.3558848798274994f, -0.8333123326301575f, 0.36477258801460266f, -6.956092834472656f, -0.47873327136039734f, -7.288418769836426f, 1.8383797407150269f, -6.37021541595459f, -1.270717740058899f, -5.504342555999756f, -7.638356685638428f, 3.2992103099823f, 3.3208279609680176f, -7.895376682281494f, -0.4653431177139282f, -5.259646415710449f, 0.3640040159225464f, -4.201903820037842f, 7.085029125213623f, 0.7169751524925232f, 1.866424798965454f, 2.98149037361145f, 2.421513319015503f, -4.835546493530273f, -3.400444746017456f, -13.251764297485352f, -7.206017971038818f, 5.5685553550720215f, 8.815764427185059f, 1.3367613554000854f, -12.028512001037598f, -4.593379974365234f, -5.485569000244141f, 7.575770378112793f, 6.186659336090088f, 4.837860107421875f, 4.19438362121582f, -2.0015146732330322f, -8.463085174560547f, 4.390081882476807f, -0.967718243598938f, -6.633697032928467f, 16.146509170532227f, -12.346258163452148f, 5.003246307373047f, 5.257402420043945f, 5.420393943786621f, -4.056494235992432f, -22.63052749633789f, -3.4521031379699707f, 0.5595429539680481f, -0.6451472043991089f, 7.716019630432129f, -3.91341233253479f, 3.537018299102783f, 0.5044690370559692f, -6.045408725738525f, -3.7082343101501465f, 8.852910995483398f, -3.696688175201416f, -2.3492298126220703f, -9.58127212524414f, -12.638714790344238f, 1.8595764636993408f, -4.140509128570557f, -7.7006120681762695f, 1.3192996978759766f, -9.62076473236084f, 9.613175392150879f, 16.121721267700195f, 7.089335918426514f, -11.461114883422852f, -9.27320384979248f, 4.465617656707764f, 1.9801151752471924f, 11.973653793334961f, -9.35599136352539f, 5.719399929046631f, -7.321558475494385f, 17.419307708740234f, -10.423422813415527f, 4.022407531738281f, -1.3151211738586426f, -5.239253044128418f, 0.22252266108989716f, -23.80808448791504f, 11.66386890411377f, -15.182354927062988f, 2.0668842792510986f, -3.095346450805664f, -10.253707885742188f, 4.4772186279296875f, -16.292877197265625f, 2.14457106590271f, 7.256747245788574f, 0.5063409805297852f, -1.0078110694885254f, -17.32522201538086f, -2.448883295059204f, -7.290302753448486f, -10.84791374206543f, 4.010873317718506f, 0.7512247562408447f, -5.4202561378479f, 2.328165292739868f, -6.0993428230285645f, -12.868612289428711f, 4.567487716674805f, -10.598518371582031f, -11.058136940002441f, -1.659664273262024f, 10.452244758605957f, 18.823354721069336f, -8.08329963684082f, 4.401593208312988f, 2.138773202896118f, -13.085124969482422f, 2.258606195449829f, -2.7691009044647217f, 5.584531784057617f, 10.423367500305176f, 2.846891164779663f, -4.120229721069336f, 1.5436748266220093f, -13.3086519241333f, -8.132325172424316f, 13.937261581420898f, -14.859661102294922f, -10.103026390075684f, 10.011719703674316f, -0.9183381795883179f, 9.600658416748047f, -9.821544647216797f, -9.450371742248535f, -2.242305278778076f, -6.559144020080566f, -4.224136829376221f, -11.820085525512695f, -6.345705986022949f, 8.94025993347168f, 4.509256839752197f, -2.219336748123169f, -0.8802061080932617f, -14.329898834228516f, 7.088708877563477f, -9.526060104370117f, 8.278675079345703f, 2.4871132373809814f, -9.620868682861328f, 9.28262710571289f, 14.8040132522583f, -19.852964401245117f, 4.130612850189209f, 5.440346717834473f, 7.126719951629639f, 0.05489916726946831f, -1.4817479848861694f, 6.829895973205566f, 12.818050384521484f, 9.886153221130371f, 1.254286766052246f, -8.354349136352539f, -11.958121299743652f, 9.16726016998291f, 4.949143886566162f, -7.31182861328125f, 18.494693756103516f, -9.817893028259277f, -0.3160404562950134f, -7.171751022338867f, 15.233163833618164f, -2.146108388900757f, 5.443892478942871f, 4.517480850219727f, 14.043611526489258f, 10.903755187988281f, -7.276175022125244f, 13.366379737854004f, -8.15054702758789f, 0.6368334293365479f, 9.139420509338379f, -6.253448963165283f, 7.538908958435059f, -4.911654949188232f, 1.2663601636886597f, 5.10722017288208f, -10.726715087890625f, -9.355097770690918f, -0.3182983100414276f, -0.6880562901496887f, 1.4052690267562866f, 8.107300758361816f, -17.92100715637207f, -4.534452438354492f, 0.5386764407157898f, 12.413888931274414f, -7.201480388641357f, -13.179725646972656f, -8.37654972076416f, -0.3154694736003876f, 3.091693878173828f, -1.0480709075927734f, 3.751927614212036f, 5.383627414703369f, 6.276157855987549f, 5.926423072814941f, -12.69626235961914f, 0.419791042804718f, 5.777501583099365f, -2.8398115634918213f, -4.607639789581299f, 6.945610523223877f, -3.02091383934021f, -9.469738006591797f, -7.620004177093506f, -2.214186191558838f, -10.362244606018066f, -3.8059444427490234f, -8.182043075561523f, 12.854665756225586f, -9.528903007507324f, 0.49294862151145935f, -8.579874038696289f, 2.118196725845337f, 7.62219762802124f, -5.029037952423096f, 12.290265083312988f, 0.5394968390464783f, 0.8183786869049072f, 1.3815820217132568f, -8.656240463256836f, -10.229368209838867f, 5.616023540496826f, 1.3402541875839233f, 0.14747254550457f, 5.180647373199463f, -5.916377544403076f, 11.535149574279785f, 6.629460334777832f, -2.8352251052856445f, 2.0122580528259277f, -0.12555190920829773f, 0.09699758142232895f, -3.735365867614746f, 6.413780212402344f, -9.827960968017578f, 4.477472305297852f, 3.9674675464630127f, 13.4703950881958f, -10.233613014221191f, 6.505621433258057f, -0.8457767963409424f, 2.9881680011749268f, -1.2331901788711548f, 0.2708655893802643f, -20.598543167114258f, -3.6357667446136475f, -10.855840682983398f, 1.586439847946167f, 10.175971031188965f, -1.9043898582458496f, 2.3536105155944824f, 4.586009502410889f, -16.701967239379883f, -5.804449558258057f, -2.78130841255188f, -12.884259223937988f, 4.541162967681885f, -2.2364847660064697f, 7.2929511070251465f, -20.930879592895508f, -1.1628355979919434f, -0.6431397795677185f, -4.187624454498291f, 9.130221366882324f, 5.2739481925964355f, -5.0708441734313965f, 0.17360350489616394f, 0.5822219252586365f, -0.22339344024658203f, -2.761117458343506f, -10.778762817382812f, 5.385908603668213f, -7.857998371124268f, 2.606891393661499f, 18.175046920776367f, -2.693230390548706f, 10.186522483825684f, -10.02297306060791f, -2.4003751277923584f, 4.620128631591797f, 11.451003074645996f, -9.571845054626465f, -7.237552165985107f, 16.154325485229492f, -19.152278900146484f, 0.16639988124370575f, -1.1797699928283691f, -3.471789836883545f, 7.653792858123779f, 3.1148579120635986f, -17.236486434936523f, 7.4174604415893555f, 5.9619293212890625f, 3.140089511871338f, -1.4843370914459229f, -11.55695629119873f, 4.42191219329834f, -1.2158066034317017f, 1.4187449216842651f, -4.891499042510986f, -0.7901540398597717f, 3.7802929878234863f, -5.735926151275635f, -6.485677242279053f, -1.5401825904846191f, 0.4863590598106384f, 7.93129301071167f, -1.8886117935180664f, 13.101932525634766f, -9.918781280517578f, 7.965061664581299f, -7.907785892486572f, -1.5377079248428345f, 13.429678916931152f, -9.150714874267578f, 11.128018379211426f, 14.368767738342285f, 3.5130884647369385f, -20.24823570251465f, 5.9857892990112305f, -16.631755828857422f, 6.948392391204834f, -0.7420387864112854f, 13.481123924255371f, -6.017175674438477f, -6.115068435668945f, -3.0654659271240234f, 13.489151000976562f, -2.2497708797454834f, 15.904927253723145f, -5.288370132446289f, -2.09615421295166f, -1.6209585666656494f, -10.90034008026123f, -2.202252149581909f, -7.51241397857666f, 1.12482488155365f, -3.7093164920806885f, 5.637340068817139f, 3.773543357849121f, 1.558279275894165f, 0.7900522947311401f, 11.120495796203613f, -4.0744709968566895f, 10.195045471191406f, -1.018953561782837f, 15.167667388916016f, 2.601929187774658f, 0.008113066665828228f, -9.717181205749512f, 6.3429789543151855f, 21.926603317260742f, -9.98530387878418f, 14.128251075744629f, 6.466258525848389f, -6.853448390960693f, -4.719650745391846f, -11.167947769165039f, -3.2867848873138428f, -12.95023250579834f, -5.806682109832764f, 11.830602645874023f, -5.1555986404418945f, -6.032625198364258f, -7.19350528717041f, -0.5742126703262329f, -20.182292938232422f, -2.5560240745544434f, 8.388191223144531f, 3.965162754058838f, -2.2216339111328125f, -11.404376983642578f, 15.09308910369873f, 8.153555870056152f, -3.3941447734832764f, -5.3842549324035645f, 17.98311996459961f, -0.8321418166160583f, -13.44106388092041f, -20.923404693603516f, -7.57475471496582f, -3.0816946029663086f, 14.614544868469238f, 3.570502519607544f, 4.2990217208862305f, 0.1919950395822525f, 1.0185343027114868f, -8.14962387084961f, 10.906466484069824f, 15.584993362426758f, -0.4128416180610657f, -6.817087173461914f, 17.120927810668945f, 2.2402291297912598f, -1.9622392654418945f, -3.413076162338257f, 6.840425968170166f, -5.948558330535889f, 6.701462745666504f, 0.8215048909187317f, -1.0856260061264038f, -3.9645702838897705f, 1.4618407487869263f, -1.7632997035980225f, 0.5018937587738037f, -2.7499217987060547f, 4.106265068054199f, -12.30443000793457f, 6.145236492156982f, 4.5076003074646f, 3.7207319736480713f, 9.945277214050293f, 1.5612727403640747f, -25.549476623535156f, -4.634391784667969f, -9.542594909667969f, 10.549148559570312f, 2.471867322921753f, 1.1155062913894653f, -2.9077959060668945f, 7.583897590637207f, -2.4130048751831055f, 8.161357879638672f, -6.786379337310791f, 15.053579330444336f, 2.3771886825561523f, 8.431512832641602f, -10.34504508972168f, 3.9834461212158203f, 0.3579562306404114f, -14.247590065002441f, -11.214945793151855f, -6.256213188171387f, 8.011175155639648f, 9.249755859375f, -11.789390563964844f, -14.64880657196045f, -0.27090367674827576f, -0.2744891941547394f, 11.454398155212402f, -14.925914764404297f, -7.099286079406738f, -3.3692843914031982f, 6.4099931716918945f, 6.6391777992248535f, 10.544389724731445f, 3.6568782329559326f, 6.840872764587402f, 12.177023887634277f, 3.4128663539886475f, -3.8373212814331055f, -0.12041311711072922f, -13.457633972167969f, 18.881927490234375f, 8.021130561828613f, -7.102904796600342f, 7.734999179840088f, 3.826378107070923f, -0.7286080718040466f, -6.966921329498291f, -9.949929237365723f, -6.5738348960876465f, -6.720236778259277f, -2.4046998023986816f, -7.772119998931885f, -3.3762123584747314f, 16.03032875061035f, -10.771323204040527f, -0.8427897095680237f, 2.035958766937256f, -9.551191329956055f, 13.150819778442383f, -8.132336616516113f, -0.9750276803970337f, 2.9933242797851562f, -12.041450500488281f, -7.16148042678833f, -7.287316799163818f, 5.334017753601074f, -7.423707962036133f, -7.503839015960693f, -13.0750150680542f, -1.5110211372375488f, -3.3898580074310303f, 6.609547138214111f, -1.1499853134155273f, 6.6649370193481445f, -4.917843341827393f, -8.266522407531738f, 16.943111419677734f, 0.35688164830207825f, 2.3017640113830566f, -2.517993927001953f, -20.720897674560547f, -5.1919379234313965f, 6.862222194671631f, 8.344823837280273f, -3.240790367126465f, -0.7951709032058716f, -7.633636951446533f, 1.9410372972488403f, -3.9540672302246094f, -8.897167205810547f, 9.550298690795898f, -12.049423217773438f, 3.2460708618164062f, -0.7465460896492004f, -7.343610763549805f, -25.89638328552246f, 1.305576205253601f, 0.7940993309020996f, 15.786218643188477f, -5.214616298675537f, 5.427506923675537f, -9.456047058105469f, -4.363546371459961f, 14.75342845916748f, -8.062644958496094f, 12.066773414611816f, -17.939416885375977f, -0.6330687403678894f, -8.203537940979004f, -15.027321815490723f, -7.841004848480225f, -6.411040306091309f, -22.517040252685547f, -4.999447822570801f, -8.532695770263672f, 8.279166221618652f, 7.497697830200195f, 8.378273963928223f, -2.62188458442688f, -3.915342330932617f, 3.5234627723693848f, -6.344028949737549f, 7.4199934005737305f, -4.750202655792236f, 1.74264395236969f, 8.607345581054688f, -10.318155288696289f, -0.5510790944099426f, 24.487709045410156f, -4.434352397918701f, 3.7080373764038086f, -16.31922149658203f, 2.130758285522461f, -1.8111369609832764f, 16.641450881958008f, -15.62347412109375f, 9.540854454040527f, -3.6384425163269043f, 1.0087566375732422f, 2.4287056922912598f, 1.3515430688858032f, -1.9539055824279785f, 7.968528747558594f, 3.1463985443115234f, 4.322893142700195f, -4.588891506195068f, 7.193646430969238f, 3.6794357299804688f, -3.6867868900299072f, -12.593381881713867f, 22.773561477661133f, -2.592869997024536f, -8.03166389465332f, 1.6411826610565186f, -5.852877140045166f, 6.320628643035889f, -1.24896240234375f, 2.633263349533081f, 2.3571205139160156f, 2.6310131549835205f, -4.19133996963501f, 9.353876113891602f, -15.862900733947754f, -11.460186958312988f, -0.3524762690067291f, 3.5821545124053955f, -4.435649871826172f, -1.1791728734970093f, 2.729346990585327f, -1.2911462783813477f, -0.1667870581150055f, 3.152031421661377f, -20.307052612304688f, -16.968524932861328f, -2.394564628601074f, -4.503590106964111f, 3.181205987930298f, 0.05041913315653801f, -0.3948255181312561f, -6.3789544105529785f, 1.732195258140564f, -5.6798787117004395f, 7.7667107582092285f, 9.2898530960083f, -4.316888332366943f, -16.857954025268555f, -5.379256725311279f, 4.286224842071533f, 4.1023759841918945f, -8.238680839538574f, 9.628629684448242f, -2.3275868892669678f, 2.8554039001464844f, -8.514399528503418f, -9.410696029663086f, 4.0246405601501465f, -6.667844295501709f, -5.36508321762085f, 0.8656852841377258f, 2.9765312671661377f, -0.6338213682174683f, -4.245971202850342f, 6.7638702392578125f, 8.446202278137207f, -2.508578300476074f, -0.5704231858253479f, 0.46178925037384033f, 10.725545883178711f, 4.313973426818848f, -6.148879528045654f, 3.38698410987854f, -12.07677173614502f, 8.890965461730957f, 9.622424125671387f, -1.9190295934677124f, 1.035591721534729f, 3.041085720062256f, 11.527141571044922f, 2.2250008583068848f, 0.06786001473665237f, 11.618230819702148f, 0.2951813340187073f, 7.107181549072266f, -13.101400375366211f, -4.043383598327637f, 5.551668643951416f, 0.23836034536361694f, -9.364093780517578f, 0.38196349143981934f, 5.709733486175537f, -3.463144540786743f, 10.42895793914795f, -21.435224533081055f, -22.471851348876953f, 2.9046216011047363f, 5.418637275695801f, -2.009808301925659f, -12.568074226379395f, 5.197458744049072f, 12.429717063903809f, -3.8452060222625732f, -8.332062721252441f, -1.3275021314620972f, 8.634135246276855f, -17.408498764038086f, 4.173489570617676f, 3.602097272872925f, 16.919105529785156f, -14.429887771606445f, -7.9446563720703125f, -15.02901554107666f, 5.9782915115356445f, -2.348747491836548f, 2.622194766998291f, 9.501197814941406f, 6.67450475692749f, 5.712249279022217f, -1.8605291843414307f, -0.874760627746582f, 15.388128280639648f, 2.725569486618042f, -4.520296096801758f, -11.915221214294434f, 3.2612195014953613f, -10.38715934753418f, 2.1763875484466553f, -1.2297403812408447f, 6.042415618896484f, -13.808868408203125f, -5.520578861236572f, -1.4444622993469238f, -7.977278232574463f, 8.31882381439209f, -9.937228202819824f, 7.792935848236084f, -8.416111946105957f, 7.689346790313721f, -4.904921054840088f, -0.9467958211898804f, -2.2417097091674805f, 4.855114459991455f, 12.061532020568848f, -4.05377721786499f, 7.5981645584106445f, -5.726215362548828f, -5.700634956359863f, -3.6235997676849365f, -8.09415340423584f, -8.389449119567871f, 3.4986400604248047f, -0.17018863558769226f, -4.8295674324035645f, 3.72519588470459f, 6.054676532745361f, 2.607694149017334f, -5.292807579040527f, 10.3839111328125f, 2.9594953060150146f, 4.974825382232666f, 30.000545501708984f, -6.096185207366943f, 0.5663076043128967f, 4.246780872344971f, -7.833420753479004f, -6.573525905609131f, -4.979223251342773f, 9.320971488952637f, 2.075157642364502f, 0.24154403805732727f, 13.304895401000977f, -9.484021186828613f, 8.853174209594727f, -7.756874084472656f, 0.7791877388954163f, 13.578595161437988f, -4.487305641174316f, -1.5067740678787231f, -20.156299591064453f, 0.3791326582431793f, -4.492989540100098f, 11.668903350830078f, -3.7837438583374023f, -0.9617695808410645f, -15.763601303100586f, 12.063730239868164f, 0.8532925248146057f, -9.071452140808105f, 3.879969358444214f, 14.453145027160645f, -2.9419867992401123f, 8.816530227661133f, 6.016716957092285f, -15.384603500366211f, 7.9487481117248535f, 5.712857723236084f, 6.595734596252441f, -8.348841667175293f, -7.271892070770264f, 8.39649772644043f, 3.010307550430298f, -7.354966640472412f, 4.326541900634766f, 6.269808292388916f, -2.21332049369812f, 2.8673322200775146f, -11.210542678833008f, 8.31998062133789f, -12.9059419631958f, 8.617777824401855f, -3.5901377201080322f, -10.140533447265625f, 0.8802559971809387f, -9.837919235229492f, -9.109603881835938f, -15.81827449798584f, 4.78360652923584f, -1.1366561651229858f, -22.160612106323242f, -12.40213680267334f, -18.063514709472656f, -2.445664882659912f, 15.82851505279541f, -4.466148853302002f, 7.4260993003845215f, 10.562188148498535f, 2.9546096324920654f, 6.616511821746826f, -11.446455001831055f, 1.037653923034668f, 8.754400253295898f, 9.139323234558105f, -6.899957180023193f, -8.12019157409668f, 10.253373146057129f, 3.2580606937408447f, -1.1452258825302124f, 6.390077114105225f, -1.9534237384796143f, -3.5342469215393066f, -6.437942981719971f, -11.188612937927246f, 1.4919053316116333f, 12.896910667419434f, 4.564462184906006f, -8.683686256408691f, 2.0561792850494385f, -7.927750587463379f, -7.966180324554443f, 3.0324785709381104f, -14.164445877075195f, -0.5540791153907776f, 7.670888900756836f, -0.11896628886461258f, 1.1451127529144287f, 22.234376907348633f, -5.0322980880737305f, 10.48377513885498f, 10.193860054016113f, -6.44713020324707f, 4.20560359954834f, -19.378570556640625f, 7.226066589355469f, -11.02578067779541f, 16.957338333129883f, 5.088015079498291f, 7.4178314208984375f, 0.37720149755477905f, 17.275354385375977f, 5.082028865814209f, 0.11316994577646255f, 10.450178146362305f}; - -// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel). -// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad -// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs. -// quad ptr for (m_tile mt, k_group g) = gemm_Apack_dram + (mt*(K/4)+g)*64. -static float gemm_Apack_dram[64*64] __attribute__((section(".data"))) = -{1.9269152879714966f, 1.4451338052749634f, 1.931160569190979f, 1.7743884325027466f, -0.685481607913971f, 0.4334380030632019f, 0.6663737893104553f, -1.3838845491409302f, -1.363769769668579f, -0.8032152652740479f, -1.2786966562271118f, 0.3220294117927551f, 0.34436315298080444f, -0.22056575119495392f, -0.7111086845397949f, -0.447700172662735f, 1.4872840642929077f, 0.8564125299453735f, 1.0118638277053833f, -0.921550989151001f, 0.5635589957237244f, -0.7171905636787415f, -0.07426456362009048f, 0.4868715703487396f, 0.1929578185081482f, -1.1209005117416382f, -0.03842746838927269f, 0.44605663418769836f, -3.1016058921813965f, 0.7118127346038818f, -0.3866667151451111f, -0.7288169264793396f, 0.9007171988487244f, 2.218075752258301f, -1.4364064931869507f, 0.9624499082565308f, -1.507175087928772f, 1.055369257926941f, -0.20960482954978943f, -1.0020296573638916f, -0.6103342771530151f, 0.1956406533718109f, 1.9138009548187256f, 1.5229592323303223f, -1.4587225914001465f, 0.3415871560573578f, 0.9578314423561096f, -0.16066236793994904f, -2.1055209636688232f, 0.5231655240058899f, -1.1298598051071167f, -0.33701515197753906f, -1.610666036605835f, -1.4533969163894653f, 0.16632132232189178f, 0.0329488180577755f, 0.16323445737361908f, -0.7815181016921997f, 0.3378446102142334f, 1.2804899215698242f, -1.4318257570266724f, 1.5885895490646362f, -0.8225308656692505f, -0.32063713669776917f, 0.6784184575080872f, 0.34664666652679443f, -0.1360345333814621f, -1.1753336191177368f, -1.4790465831756592f, 0.46515071392059326f, 1.4703037738800049f, -0.42919591069221497f, 1.51017165184021f, -1.7898789644241333f, 0.12505611777305603f, -0.11616043001413345f, -0.6071268916130066f, -0.3488781750202179f, -2.3908050060272217f, -0.6307737827301025f, -1.2345448732376099f, -0.19733144342899323f, 1.6354095935821533f, 0.35805708169937134f, 0.4322742819786072f, 0.37139150500297546f, -0.9390866756439209f, -0.9817978739738464f, 0.21230429410934448f, -0.26157355308532715f, -0.7621514797210693f, 1.3705363273620605f, -0.25973787903785706f, -0.45791950821876526f, 0.322247713804245f, -0.788766622543335f, -0.04306747764348984f, -1.0545889139175415f, 0.6547407507896423f, 0.47876790165901184f, -0.1250254064798355f, -0.004656785633414984f, -0.6013189554214478f, -0.6420586109161377f, -0.7252011299133301f, -0.44025033712387085f, -1.190559983253479f, -0.4809381365776062f, -0.7190185785293579f, -1.2322070598602295f, 1.875388741493225f, 1.3061575889587402f, -1.6046669483184814f, 1.2779955863952637f, 0.5760045647621155f, 1.353700041770935f, 0.7821183800697327f, 0.07954943925142288f, -0.09964022785425186f, 0.8265887498855591f, -0.9527732729911804f, 2.1848294734954834f, 0.7756073474884033f, -0.9903622269630432f, -0.38583096861839294f, -0.598077118396759f, 1.1042989492416382f, -0.9275763630867004f, -0.7521352767944336f, -0.1721901297569275f, 1.1415079832077026f, 0.5260620713233948f, -1.598767638206482f, 0.3781784772872925f, -0.9851518273353577f, 1.591395616531372f, 0.5216943025588989f, -0.48009708523750305f, 0.455719918012619f, -1.3641812801361084f, 0.5233525037765503f, -0.28154700994491577f, -0.5223758816719055f, -0.26273947954177856f, 1.6487230062484741f, 0.5237884521484375f, 0.018564576283097267f, 2.1120378971099854f, -0.10912995040416718f, 0.7051141262054443f, -2.488459348678589f, -0.1208132952451706f, -0.46386733651161194f, -1.2871733903884888f, 0.2503303289413452f, 0.008205652236938477f, -0.8211768269538879f, 0.05281926319003105f, -0.7401803731918335f, 0.9314952492713928f, -0.3924786448478699f, 0.056621819734573364f, -1.8058050870895386f, -0.5207571387290955f, 0.7151994705200195f, -1.7236974239349365f, -0.33131900429725647f, -0.48302069306373596f, 0.18237744271755219f, 0.7388824224472046f, -1.3610970973968506f, -0.40586018562316895f, -0.47086891531944275f, 0.42497751116752625f, 0.16235657036304474f, -0.4593467116355896f, -1.4036071300506592f, 0.4262961447238922f, 0.9254348874092102f, -0.9320061206817627f, 0.03913922235369682f, -0.8434810638427734f, 0.8435799479484558f, 0.11329790204763412f, -0.38666075468063354f, 0.03389474004507065f, 1.8018341064453125f, -0.7110859751701355f, 0.6016423106193542f, 0.4825834333896637f, -0.2369976043701172f, -0.9419456720352173f, -0.7278813123703003f, 0.575005054473877f, -0.3753443658351898f, 0.18516133725643158f, 1.305860161781311f, 0.4351435601711273f, 0.9874473810195923f, 0.0771508663892746f, -1.7906768321990967f, -0.31229403614997864f, -0.07434194535017014f, -0.3495793640613556f, -0.28251126408576965f, 0.48813387751579285f, 0.5099347233772278f, -0.7089186310768127f, -0.5594301819801331f, -0.6417241096496582f, 1.0330873727798462f, 1.0686918497085571f, 0.24659274518489838f, 0.26588720083236694f, -0.33197471499443054f, -0.9228128790855408f, 0.09329313784837723f, -0.2541753351688385f, -0.15664155781269073f, 0.3797488212585449f, 0.7692679762840271f, 1.0082393884658813f, 1.670624852180481f, 2.1860759258270264f, -0.7688388824462891f, -2.2063984870910645f, -0.6866509318351746f, 1.3065344095230103f, -1.9775909185409546f, -0.5870985388755798f, -0.8076189756393433f, -1.2619991302490234f, -1.9152568578720093f, -1.205536127090454f, -0.8708454966545105f, 0.9993040561676025f, -0.7668924331665039f, -0.595004141330719f, 1.5921050310134888f, -0.6493158936500549f, 0.7624453902244568f, -0.7508030533790588f, 0.6368136405944824f, 0.4598345160484314f, 0.01789604313671589f, 0.0826888456940651f, 0.824364185333252f, 1.0860532522201538f, -0.6421752572059631f, -0.9542103409767151f, -0.6410972476005554f, 1.2751853466033936f, -0.9494866728782654f, 0.3926331400871277f, -0.41619211435317993f, 0.45214036107063293f, 1.6423169374465942f, 0.01086814422160387f, -0.9726738929748535f, -0.8146268725395203f, -1.3793021440505981f, 0.8853808045387268f, 0.024699924513697624f, 1.096641182899475f, 1.3438509702682495f, 0.061276569962501526f, -0.414562851190567f, 0.9594927430152893f, 0.01691739819943905f, 0.8229668736457825f, 1.861944556236267f, 0.8520749807357788f, -0.1595974713563919f, -0.33874234557151794f, 0.9584577679634094f, -1.0212392807006836f, 0.625802755355835f, 0.1824439913034439f, -1.0641486644744873f, -0.6836934685707092f, -1.2922308444976807f, 0.08526104688644409f, -0.6902380585670471f, 0.10350999981164932f, 0.08027740567922592f, -0.886031985282898f, -1.077892780303955f, -1.6946725845336914f, -0.4973975419998169f, -1.3406795263290405f, 1.6192004680633545f, -0.49492356181144714f, -2.5849502086639404f, 0.7863810062408447f, -0.7601934671401978f, 0.06604336202144623f, 0.766244113445282f, 0.7481252551078796f, -0.22995619475841522f, 0.8290349841117859f, 0.7448412775993347f, 1.4801039695739746f, 0.8848565220832825f, 1.1805996894836426f, 0.439589262008667f, -0.5853705406188965f, 1.450609803199768f, -0.5922516584396362f, -0.02399955503642559f, -0.057920295745134354f, -0.4075061082839966f, -0.0007737990817986429f, 0.64540034532547f, -0.16356196999549866f, -2.172283887863159f, 2.0921294689178467f, 1.345484972000122f, 0.8391514420509338f, -0.8342104554176331f, -2.8929238319396973f, -0.7581311464309692f, 0.5361881256103516f, 0.2694815397262573f, 0.15431594848632812f, -0.1221928745508194f, 0.5666652917861938f, 0.9623646140098572f, 0.1620604395866394f, 0.353316068649292f, -0.9085567593574524f, 0.08768323808908463f, 0.7953095436096191f, 0.12682189047336578f, -0.20004984736442566f, 1.0300744771957397f, -0.3875778615474701f, 1.078317642211914f, 0.5246226191520691f, -0.21037597954273224f, 0.4407670795917511f, -0.7469954490661621f, -0.7097623348236084f, -0.14264194667339325f, 1.195958137512207f, -2.6474881172180176f, 0.3129958212375641f, 1.0937845706939697f, 0.2792847752571106f, -2.4520716667175293f, 0.9949536919593811f, -0.8680985569953918f, -0.7124031782150269f, 0.8008005619049072f, 1.1412016153335571f, -0.7328027486801147f, -0.14829230308532715f, 1.7093087434768677f, -0.4875054359436035f, 0.15271379053592682f, -1.3061535358428955f, -1.4575366973876953f, 0.8050477504730225f, -0.1177205815911293f, 0.1864478439092636f, 0.4159761369228363f, 0.7201864719390869f, -0.5701602697372437f, -1.6171332597732544f, 1.680620551109314f, 0.05164359509944916f, 0.10429783165454865f, -2.3184432983398438f, 0.05792269483208656f, 0.050095997750759125f, -0.0388023778796196f, -1.4039719104766846f, -0.9712379574775696f, -1.1133604049682617f, -0.29864323139190674f, 0.3547132909297943f, 1.9025356769561768f, -0.13413065671920776f, 0.32332202792167664f, -0.35899198055267334f, 1.27912437915802f, 0.7439519762992859f, 0.3487516939640045f, -0.39799532294273376f, 1.1929805278778076f, 0.6084084510803223f, 0.9446058869361877f, -1.0597201585769653f, 0.25403109192848206f, 0.4981626570224762f, -0.9536206126213074f, 0.09063850343227386f, -0.7346699833869934f, -1.4067999124526978f, 1.1284750699996948f, 0.051366694271564484f, 1.2964228391647339f, -0.4815843999385834f, 0.9675941467285156f, 1.0804862976074219f, 1.9372931718826294f, 1.6308681964874268f, -1.5824053287506104f, 0.3057299852371216f, -0.1790592074394226f, -1.1999552249908447f, -0.09247277677059174f, 1.7422553300857544f, 0.044657133519649506f, -2.3609628677368164f, -1.2123126983642578f, 0.6950237154960632f, 0.610466480255127f, -1.0494661331176758f, -0.46568843722343445f, -1.7808643579483032f, 0.7287133932113647f, -0.08472306281328201f, 0.9871290922164917f, 0.4150581359863281f, 1.1992844343185425f, 0.12711313366889954f, -1.01665461063385f, -1.2660012245178223f, -1.5211198329925537f, -0.2904934287071228f, 2.602391004562378f, 1.835181474685669f, 1.334737777709961f, 0.603898823261261f, 1.6047972440719604f, 1.5080454349517822f, 0.9808937907218933f, 1.0844124555587769f, 1.1456739902496338f, -0.7174144983291626f, -0.4292171895503998f, 0.4403660297393799f, -0.007675689645111561f, 0.38916081190109253f, 0.3478375971317291f, -0.13345853984355927f, -0.09572362899780273f, -1.9180361032485962f, -0.2316243201494217f, -1.7222950458526611f, -2.4801201820373535f, 0.30942854285240173f, 0.41459226608276367f, 0.9477656483650208f, -0.14181147515773773f, 2.833967924118042f, 1.010284185409546f, 0.6377718448638916f, -0.518220841884613f, 0.34287506341934204f, 0.7401772737503052f, -0.15693345665931702f, -0.08114803582429886f, -1.3923954963684082f, 0.041759490966796875f, -0.827768862247467f, -0.4175437390804291f, -0.5003090500831604f, 1.15656316280365f, -0.676629364490509f, -0.2763414680957794f, 1.9534740447998047f, 0.6110401153564453f, 0.15978877246379852f, 0.83954256772995f, -1.4590637683868408f, 1.4161995649337769f, 1.138344645500183f, 1.2586976289749146f, 0.540465772151947f, -0.2515752911567688f, 1.334702968597412f, -1.1954537630081177f, 1.0350031852722168f, 0.2690545618534088f, -0.5730168223381042f, -0.19321373105049133f, 2.0486814975738525f, 1.2208385467529297f, 1.7697970867156982f, 0.05852266773581505f, -1.4936561584472656f, 0.6833979487419128f, -0.2505214214324951f, 0.8691263794898987f, 0.4350730776786804f, 0.859858512878418f, 0.48353928327560425f, 0.8123369216918945f, 1.6896470785140991f, -0.036629438400268555f, -0.3303174376487732f, 0.7767809629440308f, -1.0880382061004639f, -0.6076440215110779f, 0.6268176436424255f, -1.6682480573654175f, -0.22138521075248718f, -0.13825182616710663f, 1.6704555749893188f, -0.9609367251396179f, -2.2717032432556152f, -1.3846737146377563f, -2.5095443725585938f, -1.9005532264709473f, -0.004505051765590906f, 0.9732939004898071f, -0.7939430475234985f, 0.6838752627372742f, 1.621694564819336f, -1.7376028299331665f, -1.8736529350280762f, 2.129624843597412f, 0.22523505985736847f, 0.9212995171546936f, -0.545271098613739f, 0.05182264745235443f, -0.13386189937591553f, -0.8712361454963684f, 0.4880010485649109f, 0.22857652604579926f, 1.666792392730713f, -1.0150787830352783f, 0.3752319812774658f, -1.3245893716812134f, 0.8512656688690186f, -0.12535162270069122f, 2.3259060382843018f, -1.5181471109390259f, -0.07724537700414658f, 0.5282443761825562f, -2.15816330909729f, -0.3284812867641449f, -0.058557309210300446f, -0.223365917801857f, 0.7845868468284607f, 0.02485940419137478f, 0.15392017364501953f, -0.5419175624847412f, 0.08790969103574753f, -0.5160817503929138f, -0.40046969056129456f, -1.3658148050308228f, -0.9203909635543823f, 0.1387282907962799f, 0.9856945276260376f, -0.008228386752307415f, -1.6607975959777832f, -2.247206211090088f, 0.12574470043182373f, 1.7173614501953125f, 0.02864718623459339f, -0.34595024585723877f, -1.0602530241012573f, -0.44102486968040466f, -1.241483449935913f, 0.6001842617988586f, -0.6088271737098694f, 1.111746072769165f, 0.6661149263381958f, -1.1797568798065186f, 1.2783364057540894f, -1.4493319988250732f, -0.6637441515922546f, -0.4478967487812042f, -0.5525766611099243f, 0.3188803195953369f, 0.640755295753479f, 0.2868328094482422f, -0.572657585144043f, -0.3136177957057953f, -0.32025346159935f, -0.4702208340167999f, -0.508095383644104f, -0.6227966547012329f, -0.44026491045951843f, -0.5297411680221558f, 0.28815189003944397f, -0.605182409286499f, 0.3657907545566559f, 0.4234687089920044f, 0.07448001205921173f, -0.42451897263526917f, 0.5832474231719971f, -0.7308424115180969f, 0.0835680365562439f, -0.12925422191619873f, -0.844377875328064f, -0.6086435317993164f, -0.6184902191162109f, -0.7891808748245239f, -2.3179564476013184f, 0.9625157713890076f, 0.869049608707428f, -0.17924511432647705f, -0.39920157194137573f, -0.3874586224555969f, -0.1492866724729538f, 0.3057209253311157f, 1.0669267177581787f, 0.17482025921344757f, 0.39990535378456116f, -0.7149624228477478f, -0.5513465404510498f, -0.046192023903131485f, -1.647040605545044f, -0.167823925614357f, 1.294582724571228f, 0.2794382870197296f, -0.8097057938575745f, 0.19955870509147644f, 0.49674081802368164f, -0.22963792085647583f, -0.5522539615631104f, -0.7745925188064575f, -0.4501533806324005f, -1.0939292907714844f, 1.989207148551941f, -0.047562163323163986f, 1.9889612197875977f, -1.6457397937774658f, -1.0362098217010498f, 1.6433145999908447f, 0.22267311811447144f, -0.5718191266059875f, -1.4298604726791382f, -1.2461947202682495f, -2.369169235229492f, -0.40709349513053894f, -0.09342008084058762f, -1.5575724840164185f, -0.18526747822761536f, -1.6021603345870972f, -0.07198750972747803f, 2.0207436084747314f, 1.900311827659607f, -0.4833274185657501f, -0.4503057301044464f, 2.0070879459381104f, -0.8483412265777588f, -2.7936289310455322f, 0.45901596546173096f, -0.41459938883781433f, -0.5614708065986633f, 0.8702965974807739f, -1.0284309387207031f, 0.9956361055374146f, 0.7527588605880737f, 1.3528969287872314f, -0.906094491481781f, 0.25391900539398193f, 1.6950805187225342f, -0.740294337272644f, -0.0729660615324974f, -1.2531019449234009f, 1.6489421129226685f, -0.7111545205116272f, 0.5309328436851501f, 1.4558700323104858f, -0.5949130654335022f, -1.0552809238433838f, 0.40444278717041016f, -0.8797858357429504f, 0.4047577977180481f, 1.288827657699585f, -2.0487122535705566f, 0.9364385008811951f, 0.028089528903365135f, 0.31428107619285583f, -0.5479549169540405f, 1.118869423866272f, 1.6005686521530151f, 0.5235219597816467f, -1.3614802360534668f, 0.3316534161567688f, 1.2687278985977173f, -1.3284013271331787f, 2.1425962448120117f, -0.6011420488357544f, 0.17846599221229553f, 0.05229547247290611f, -1.0810555219650269f, 0.7122363448143005f, -0.17536965012550354f, 0.1415553092956543f, -1.1425532102584839f, 1.7732776403427124f, -0.07858924567699432f, -1.71055006980896f, 1.9562491178512573f, -1.00010085105896f, 1.2904434204101562f, 0.7060741186141968f, -0.5153723955154419f, -1.2741512060165405f, 0.2649095058441162f, -1.5468504428863525f, 0.01762307994067669f, -0.031765542924404144f, -1.7734959125518799f, 1.0348176956176758f, -0.44875210523605347f, -2.071660280227661f, 0.43104586005210876f, 0.8384853601455688f, 1.7684898376464844f, -0.6919524669647217f, -1.1755682229995728f, 0.35730111598968506f, 1.0827196836471558f, 2.1227850914001465f, 1.2731683254241943f, 0.7567060589790344f, 0.0782259851694107f, 0.10164086520671844f, -0.7046414017677307f, -0.626437783241272f, -0.03045438416302204f, -0.4125255346298218f, 0.3683530390262604f, -0.2698453664779663f, -0.9857985377311707f, -0.47199076414108276f, -0.0783226415514946f, 0.5892837643623352f, 1.2498642206192017f, -1.234653115272522f, -0.0013108636485412717f, 0.7755194902420044f, 0.19315829873085022f, 1.3433040380477905f, -0.39465200901031494f, -0.5150922536849976f, 0.3830311596393585f, -0.9769555926322937f, 0.7637977004051208f, 0.12306158244609833f, -1.2370758056640625f, -1.2894343137741089f, -0.9705761075019836f, 0.9187757968902588f, 0.9821351766586304f, -0.4879138767719269f, -0.30360376834869385f, 2.0265355110168457f, 0.40967342257499695f, 0.7132695913314819f, 1.8868111371994019f, 0.6902899742126465f, -0.04476971551775932f, -0.03363388776779175f, 1.1792222261428833f, 0.8757511377334595f, -2.301875114440918f, 1.0762810707092285f, 1.4723693132400513f, 0.6662830114364624f, 0.22690092027187347f, -0.9138230085372925f, -1.457029104232788f, 0.03581761196255684f, -0.9291303157806396f, 0.4038029611110687f, -0.21844321489334106f, -0.4939993619918823f, 1.179942011833191f, 1.8594977855682373f, -0.4137862026691437f, 0.15132997930049896f, -0.0010087001137435436f, -1.0667427778244019f, 1.4108561277389526f, 0.24650610983371735f, 0.4927920699119568f, -0.6581372618675232f, -0.10233523696660995f, 0.12058872729539871f, 0.2761908769607544f, -0.7139783501625061f, 0.16629981994628906f, 1.1366126537322998f, -0.33142781257629395f, 2.6221468448638916f, 0.5184088349342346f, 0.739393413066864f, -0.8494256734848022f, -1.9893426895141602f, -1.3143675327301025f, 0.1328691989183426f, -0.5128253102302551f, 0.07802387326955795f, -0.5991530418395996f, -0.8056637048721313f, -0.5388752818107605f, 0.8337291479110718f, 2.1441681385040283f, -0.46184006333351135f, 0.6495042443275452f, 0.36905255913734436f, -0.7015367746353149f, 0.27310314774513245f, -1.6593921184539795f, 0.29731303453445435f, -1.31621515750885f, 0.12191462516784668f, 0.3006223440170288f, 0.5258087515830994f, 0.4770564138889313f, -0.20757682621479034f, 0.4625823199748993f, -0.9585452079772949f, 1.7045671939849854f, 1.419979453086853f, 0.09495851397514343f, 0.3802972435951233f, -0.4323408901691437f, 2.7312309741973877f, 0.3062905967235565f, 0.4344584047794342f, -1.2524477243423462f, 0.47808775305747986f, 0.07734657824039459f, -0.48799172043800354f, 0.7261772155761719f, -0.9319478273391724f, -0.8718891143798828f, 0.45363426208496094f, 0.3459012508392334f, 0.848518967628479f, -0.7525874376296997f, 0.19898031651973724f, 0.1414770483970642f, 0.43200522661209106f, 1.182044506072998f, 0.0033933203667402267f, -1.5844100713729858f, 0.2761341631412506f, 0.6477669477462769f, 1.1913690567016602f, 0.09115186333656311f, -1.5909662246704102f, -0.027118360623717308f, 1.2460919618606567f, 0.6424751281738281f, -0.047891248017549515f, -0.647229790687561f, -0.23609064519405365f, 0.07110362499952316f, -0.30918216705322266f, 0.32602694630622864f, -1.0240145921707153f, -2.5446670055389404f, -0.5895728468894958f, -0.4324244260787964f, -0.8140076398849487f, -0.3890652060508728f, -1.13597571849823f, -0.3532457649707794f, -2.3065085411071777f, -0.20395424962043762f, 0.6685602068901062f, -1.2822614908218384f, 0.30340856313705444f, 0.5633530616760254f, -0.09658124297857285f, -0.3894469738006592f, 0.22404761612415314f, 1.3719074726104736f, 0.569182813167572f, 1.1740480661392212f, -0.7359927892684937f, 0.5279164910316467f, -0.52259761095047f, 1.4638569355010986f, -1.2868918180465698f, 0.6853673458099365f, 1.0429801940917969f, 1.96529221534729f, -0.45007675886154175f, -0.5786357522010803f, 1.541925072669983f, 2.8543806076049805f, -0.7554785013198853f, -0.5379461050033569f, -0.7911050319671631f, 0.7011352777481079f, -1.4032478332519531f, -0.012685478664934635f, -0.5187733173370361f, 1.255434274673462f, 0.17988650500774384f, -0.13968797028064728f, 0.6899017095565796f, -0.9638485312461853f, 0.47390419244766235f, -1.083811640739441f, -0.108744777739048f, 0.8243650794029236f, 1.3675810098648071f, 0.7378400564193726f, -0.19896702468395233f, 0.6674330234527588f, 0.03600366786122322f, 0.24083632230758667f, -1.5012763738632202f, -0.7149558067321777f, -2.126762628555298f, -1.1807502508163452f, -1.3129348754882812f, -2.5667941570281982f, 0.6503364443778992f, -0.3889259994029999f, -0.4189043343067169f, 0.7983470559120178f, -0.3197358250617981f, -0.8505349159240723f, -1.3615714311599731f, -0.8035953640937805f, -0.06347727030515671f, 0.13253536820411682f, -1.9266542196273804f, 0.8539193272590637f, -0.13408313691616058f, -1.282929539680481f, 0.03780364990234375f, 0.7096128463745117f, 1.1662380695343018f, 0.8126105666160583f, 1.4384385347366333f, 1.8890222311019897f, -0.9130924344062805f, 0.03610055148601532f, -0.5193602442741394f, -1.3776048421859741f, 0.6756148934364319f, 0.7642406225204468f, 0.1278512328863144f, 0.512991189956665f, -1.0407685041427612f, 0.448485791683197f, -1.1702114343643188f, 0.8198426961898804f, 0.01693599671125412f, 1.4981187582015991f, -0.7068426609039307f, 0.5934628248214722f, 1.919209361076355f, 1.3406710624694824f, 0.07648162543773651f, -0.4410470724105835f, -0.0978068932890892f, 1.095009684562683f, 1.0229133367538452f, 0.5397310256958008f, -0.7647228837013245f, -0.590737521648407f, -0.10318559408187866f, 0.6214459538459778f, 0.5325868129730225f, 0.043896086513996124f, -1.2519514560699463f, 0.0696544423699379f, -1.6514869928359985f, 0.9199973940849304f, 0.34005025029182434f, 0.1417587399482727f, 1.8445940017700195f, 0.3398909568786621f, -0.5557951331138611f, 0.5655050277709961f, -0.05528254434466362f, 0.8540631532669067f, 1.1894739866256714f, 0.42318588495254517f, -0.6035352349281311f, 1.4443233013153076f, 3.0250484943389893f, -1.6034338474273682f, 2.1477253437042236f, -0.3787555396556854f, 1.4557304382324219f, 1.1084681749343872f, -1.184537410736084f, 0.7199674844741821f, 0.7042727470397949f, 0.5057917237281799f, 1.204850673675537f, -0.4900680184364319f, 0.7606944441795349f, -0.33889833092689514f, -0.1742597371339798f, 0.23202891647815704f, 1.3462589979171753f, -0.42982181906700134f, -0.6604134440422058f, -1.5597758293151855f, -0.3461014926433563f, 0.5544233322143555f, 1.3835493326187134f, 0.41140761971473694f, 0.7098760008811951f, 0.22245365381240845f, -0.982473611831665f, -0.35945725440979004f, -0.7463049292564392f, 0.5179733633995056f, 0.6092063784599304f, 0.5064983367919922f, 0.8556069731712341f, 0.5761587619781494f, 0.11352583765983582f, -0.8009540438652039f, -0.2633814215660095f, 1.5817502737045288f, -1.2247875928878784f, 0.23484112322330475f, -1.407819390296936f, -0.09603477269411087f, -0.294853538274765f, -0.5165784358978271f, 0.2704370319843292f, -1.0562596321105957f, -0.8496796488761902f, -0.2919580638408661f, 0.10698884725570679f, -0.9970720410346985f, 0.05950453504920006f, 0.6107109189033508f, 0.5484238862991333f, 0.7350953221321106f, 0.962885856628418f, 0.08861476927995682f, -0.0801108106970787f, 1.0644340515136719f, -0.2798626720905304f, -0.6881742477416992f, -1.1818499565124512f, 0.24130821228027344f, -0.6987038254737854f, 0.45643651485443115f, -0.10149910300970078f, -0.34807199239730835f, 1.7285562753677368f, 0.2142704576253891f, 0.45765289664268494f, -0.269319087266922f, -1.5785412788391113f, -0.3476867079734802f, 0.5194124579429626f, 1.4888246059417725f, 1.0837124586105347f, 0.4755038022994995f, -1.0340323448181152f, 0.18275369703769684f, -0.20516234636306763f, -0.3147018849849701f, 0.08301469683647156f, 0.2176828533411026f, 0.2920781075954437f, -0.17470814287662506f, 0.9676891565322876f, 0.45085909962654114f, 0.6715953946113586f, 0.8490674495697021f, 1.170888900756836f, 0.8825610876083374f, 0.17298388481140137f, -1.4316335916519165f, -0.4947497546672821f, 0.6246524453163147f, -0.7811664342880249f, -0.41328999400138855f, 0.712009072303772f, 1.1277714967727661f, -0.6925867199897766f, -1.7561272382736206f, -0.6673583388328552f, 0.04091703146696091f, -0.060151856392621994f, 0.2014705091714859f, 2.1779797077178955f, -0.23840203881263733f, 0.5123522281646729f, 0.1427735835313797f, -0.8951197266578674f, -0.7939775586128235f, 0.6872723698616028f, 0.3946205973625183f, -0.9005926847457886f, -1.5005147457122803f, -0.8442767858505249f, 1.4259073734283447f, 1.6183183193206787f, -0.06933283060789108f, 0.06978437304496765f, 0.3839779198169708f, 1.7791979312896729f, 0.5468734502792358f, -0.9818529486656189f, 0.6328914165496826f, 0.5340191125869751f, -0.674835205078125f, 0.7835897207260132f, 1.1304746866226196f, 0.8906894326210022f, -0.2404831200838089f, -0.32920387387275696f, 0.5127183794975281f, -0.26440465450286865f, 0.2585761547088623f, -1.6634936332702637f, 1.2309634685516357f, 0.25832492113113403f, -0.06058019772171974f, 1.1258721351623535f, -1.048923373222351f, -0.88661128282547f, -0.3876877427101135f, -1.1108732223510742f, 0.8258382678031921f, 0.47655177116394043f, -0.4854878783226013f, -0.1140289157629013f, -0.4026731848716736f, 1.354137897491455f, 0.4775455594062805f, -0.7650561332702637f, 1.2286686897277832f, -2.4340736865997314f, -0.5304896831512451f, 0.25538599491119385f, -0.5224623084068298f, 0.7963698506355286f, 0.44965043663978577f, -3.106328010559082f, 0.9458276629447937f, -0.8396266102790833f, -0.06616523861885071f, -0.8452204465866089f, 1.9770677089691162f, -0.07092823088169098f, -1.4889180660247803f, 1.2306435108184814f, 0.7042104005813599f, -0.3497500419616699f, -2.0363707542419434f, -0.4588965177536011f, -1.1338030099868774f, 0.6588250994682312f, 0.3726101517677307f, -0.9897713661193848f, -0.15447334945201874f, 0.33320167660713196f, -0.9029294848442078f, 0.3004419207572937f, 0.0267170500010252f, -0.3697128891944885f, -1.2930028438568115f, 0.4252126216888428f, -0.05628490820527077f, -1.338055968284607f, 0.5246880650520325f, -0.9283785820007324f, -0.14128278195858002f, -0.2966694235801697f, -1.9104946851730347f, -0.6022037863731384f, -1.6013476848602295f, -1.2525506019592285f, 0.6440190076828003f, 1.6395184993743896f, -0.25020739436149597f, -0.24597389996051788f, 0.8795658946037292f, -0.016383398324251175f, -1.4897207021713257f, -0.4389103353023529f, -0.6970252990722656f, -0.17175325751304626f, -0.6456266641616821f, 0.23320063948631287f, 0.26085028052330017f, -0.7153372168540955f, -0.059470854699611664f, -0.5745509266853333f, 0.7591805458068848f, -1.0744175910949707f, -0.09136287868022919f, 0.3293827772140503f, 0.07306429743766785f, -0.10749480873346329f, -1.5194628238677979f, -0.5850174427032471f, -0.08793152123689651f, -0.6866653561592102f, 0.4101375937461853f, -1.3473118543624878f, 1.4177610874176025f, -0.467404842376709f, -0.9928666353225708f, -1.9059150218963623f, -2.0202691555023193f, 0.32121846079826355f, -0.5283262729644775f, 0.6973733305931091f, 0.9967362284660339f, -1.3085604906082153f, 0.32580918073654175f, 1.8071491718292236f, -0.27431318163871765f, -0.1326882690191269f, 0.32671934366226196f, -0.8231000304222107f, 0.6738032102584839f, 0.551419734954834f, 1.1634254455566406f, -0.9665390253067017f, -0.6739510893821716f, 0.2892301678657532f, -0.46433013677597046f, 0.8342204093933105f, 1.0237910747528076f, 0.659813642501831f, -1.458429217338562f, -0.7326241135597229f, 1.2922906875610352f, 1.6295740604400635f, -0.8344282507896423f, -0.539470374584198f, 1.4665507078170776f, 2.654942035675049f, 1.609459400177002f, 0.367727130651474f, -0.9192121624946594f, 0.35196706652641296f, -0.15669254958629608f, -1.098692774772644f, -1.2132517099380493f, -0.07032525539398193f, 1.8989076614379883f, 0.4093967378139496f, -1.4458993673324585f, -1.5456795692443848f, -0.4921732246875763f, 0.09215065091848373f, -1.1077474355697632f, 1.0582451820373535f, -0.29416555166244507f, -0.5785751342773438f, 1.212047815322876f, 2.0987792015075684f, -1.5964131355285645f, 0.5674765110015869f, 0.9744538068771362f, 0.27448296546936035f, -0.04056643322110176f, -0.5840954780578613f, -0.31466683745384216f, -0.16959930956363678f, 0.658042311668396f, 0.8539144992828369f, -0.7443782091140747f, -0.1468161642551422f, 1.0819309949874878f, 1.2373405694961548f, -1.3463493585586548f, -0.5285679697990417f, -1.5322294235229492f, 0.8283826112747192f, 0.45322731137275696f, -0.3450125455856323f, -0.2933650016784668f, 0.10613418370485306f, 0.11260014772415161f, 0.02781728096306324f, 0.5361921191215515f, -1.0331605672836304f, 1.0188977718353271f, -0.8913255929946899f, 0.8866236805915833f, 0.8713390231132507f, -0.4831593930721283f, -1.7715193033218384f, 0.8562206029891968f, -3.832531690597534f, -0.9927570819854736f, -0.11961783468723297f, 1.397810459136963f, -0.3067088723182678f, -1.4679176807403564f, 0.0910743772983551f, 1.2350000143051147f, 0.5837976336479187f, -1.8317182064056396f, 0.1937909871339798f, -0.8611426949501038f, -0.522757887840271f, 1.7186238765716553f, 0.0968942642211914f, 0.4322175085544586f, 0.11584769189357758f, -0.2557562291622162f, 1.1861584186553955f, -0.9166569709777832f, 0.8642276525497437f, -1.716816782951355f, 0.6718529462814331f, -0.21214154362678528f, 1.0166347026824951f, 0.2806217670440674f, 1.9681813716888428f, -0.27264782786369324f, 1.2400078773498535f, -0.5684311389923096f, 0.36735019087791443f, 0.24113479256629944f, 1.9915446043014526f, -0.9585699439048767f, -1.2203160524368286f, -0.7793720364570618f, -1.0658658742904663f, -0.5502451658248901f, 0.9851812124252319f, 1.387345790863037f, -0.794191300868988f, 0.6909231543540955f, -0.7403607368469238f, 0.9804211258888245f, -0.905766487121582f, -2.9151322841644287f, 0.47318676114082336f, -0.05469474941492081f, 1.0238094329833984f, -0.5702749490737915f, 0.2909986078739166f, -0.41753849387168884f, -1.0129939317703247f, 0.535078763961792f, -0.7609738707542419f, -0.8248465657234192f, 0.36742645502090454f, 0.7183824777603149f, -0.8665743470191956f, -0.17533333599567413f, 0.768028736114502f, 1.0834310054779053f, 0.5876799821853638f, 0.04771281033754349f, 2.1327197551727295f, 0.19281719624996185f, -0.07964225858449936f, 1.1059718132019043f, -0.9939178228378296f, -1.3392163515090942f, -1.2726119756698608f, 0.3544987440109253f, 0.42930465936660767f, -0.5719326138496399f, -0.30639445781707764f, -0.12276917695999146f, 1.62221360206604f, 0.07731132954359055f, 0.183979332447052f, -0.8637551665306091f, 0.23347528278827667f, 0.3847457766532898f, 1.3200364112854004f, 0.2528532147407532f, 2.9082677364349365f, 1.2357676029205322f, -0.6267421841621399f, -0.28073710203170776f, 2.0288591384887695f, -0.46663370728492737f, -0.5359372496604919f, 0.7414069175720215f, 0.08158037811517715f, 1.2316521406173706f, -0.8425887823104858f, -1.141897201538086f, 1.2004878520965576f, -0.8595814108848572f, -1.5196866989135742f, -0.10754015296697617f, 1.4483332633972168f, -2.0370566844940186f, 1.3712586164474487f, -0.243259459733963f, -0.7683921456336975f, 0.1017654687166214f, -0.35750812292099f, 0.35395190119743347f, 0.20281589031219482f, 2.9194436073303223f, -1.6971954107284546f, -0.8292406797409058f, -1.2668757438659668f, 1.2814747095108032f, -0.29335519671440125f, 0.7705280780792236f, -0.5614521503448486f, 1.4171453714370728f, 0.23598231375217438f, -0.29366371035575867f, -1.2262547016143799f, 0.38033702969551086f, -1.2398927211761475f, -0.5345798134803772f, 0.33024173974990845f, 1.9377914667129517f, 1.0871516466140747f, 1.3149042129516602f, -0.9444983005523682f, 0.1276102066040039f, 2.106604814529419f, -1.1304327249526978f, -0.9464563131332397f, 0.1686755269765854f, -0.446566104888916f, -0.6286743879318237f, -0.11334973573684692f, -1.962886095046997f, -1.523536205291748f, 0.645367443561554f, -0.9533721804618835f, -0.5536230206489563f, 0.6801379919052124f, 1.2470786571502686f, -0.1793764978647232f, 0.6101059317588806f, -0.10875027626752853f, 0.996456503868103f, -0.7419731020927429f, -1.1421011686325073f, -1.1778020858764648f, -0.04226749762892723f, 0.30849045515060425f, -0.7805798053741455f, -0.8158796429634094f, -2.9890658855438232f, 1.5734566450119019f, -1.3029974699020386f, 1.1616994142532349f, -0.25818052887916565f, 0.31204938888549805f, -1.2942535877227783f, 0.608341634273529f, -1.1809622049331665f, 0.15562251210212708f, 0.6069639921188354f, 1.4125137329101562f, -0.27004849910736084f, -0.4410642385482788f, -0.1343953162431717f, 1.1372771263122559f, 0.18370741605758667f, 1.8697383403778076f, 1.0696011781692505f, -0.17657233774662018f, -1.340453863143921f, 0.717379093170166f, 0.6038413643836975f, 0.7894347310066223f, 0.9626035690307617f, -0.25843867659568787f, -0.8331825137138367f, -0.02316661737859249f, 1.4387904405593872f, -0.77969890832901f, -0.3609391152858734f, 0.21925963461399078f, -0.4726978540420532f, -1.0638948678970337f, -0.4561823606491089f, 0.5214431285858154f, -0.5561836957931519f, -0.2302778661251068f, 1.117246150970459f, 0.7824702858924866f, -1.1049346923828125f, -0.7501540780067444f, -0.47921040654182434f, -0.011093219742178917f, 0.03258634731173515f, -0.756567120552063f, 0.104627326130867f, 0.4133651852607727f, -0.958824634552002f, -0.2272576242685318f, 1.3634916543960571f, -2.3571479320526123f, 1.173104166984558f, 0.7147674560546875f, 0.4163925051689148f, -0.06465863436460495f, -0.7909473180770874f, 1.2354754209518433f, 0.2998451292514801f, -0.9952824711799622f, -0.5479734539985657f, 1.514818549156189f, -0.325872004032135f, 0.0061524491757154465f, -1.5123530626296997f, 0.25006136298179626f, -2.4219810962677f, -0.8351162672042847f, 0.6859942674636841f, 1.443666696548462f, 0.6747081875801086f, -0.0002302070497535169f, -0.21609316766262054f, 1.0141247510910034f, 0.721377432346344f, -0.29935362935066223f, -0.49368005990982056f, -1.5796416997909546f, 0.3189202845096588f, -0.5672794580459595f, 1.5067633390426636f, 1.161847710609436f, -0.08308011293411255f, -2.262038469314575f, 0.8626111745834351f, 0.196146160364151f, 0.2249874323606491f, 0.6830949187278748f, 0.0019485306693241f, 1.01323664188385f, -0.6184468865394592f, 0.7670295834541321f, 2.8818862438201904f, 0.6386357545852661f, -0.10977188497781754f, -0.17038141191005707f, -0.938051164150238f, -0.11422315984964371f, 0.1034913882613182f, -1.2966164350509644f, -0.41023650765419006f, -0.8332573175430298f, -0.9556332230567932f, 0.1063748151063919f, -0.20979070663452148f, 0.6346396803855896f, 0.545662522315979f, -0.937210738658905f, -1.1672022342681885f, -0.4361383616924286f, 0.0964970588684082f, -0.3027929663658142f, -0.638506293296814f, -0.05629456788301468f, 0.11661393195390701f, 0.3269132375717163f, -0.7588294148445129f, -0.952683687210083f, -0.7779799103736877f, 0.3503226041793823f, 1.2010222673416138f, 0.8768793344497681f, -0.7691330313682556f, -2.330476999282837f, 1.9413354396820068f, -1.0022987127304077f, -1.4932167530059814f, -1.2868070602416992f, 0.21946865320205688f, 0.8497498631477356f, -0.02522038109600544f, 0.6064606308937073f, 1.6981608867645264f, -0.36366915702819824f, 0.6935068964958191f, 0.12109924107789993f, 0.675596296787262f, 0.8142848610877991f, 0.0793362557888031f, -0.7808834314346313f, -1.163609266281128f, 0.3780289888381958f, 0.5237964987754822f, -1.3662828207015991f, -0.439242959022522f, -0.8599120378494263f, 0.3788713812828064f, -0.46068817377090454f, 0.7436892986297607f, -1.2479252815246582f, -0.4358613193035126f, 0.2984321117401123f, -1.8900177478790283f, 0.19737061858177185f, -0.7584667801856995f, 0.8250064849853516f, -1.5966553688049316f, 0.04276036471128464f, 0.7530690431594849f, -0.04625223949551582f, -0.1391070932149887f, -0.6105663776397705f, 0.24455690383911133f, -0.8800807595252991f, -0.03276701644062996f, -0.09261447191238403f, -0.9824548959732056f, 1.3447729349136353f, 0.19431965053081512f, -0.6367602348327637f, 0.941990315914154f, 1.2206652164459229f, 0.08320564776659012f, 0.5858650803565979f, -0.2219216674566269f, -0.6149584650993347f, -0.018692156299948692f, 1.0629346370697021f, -0.8915789723396301f, -1.4766337871551514f, 1.0600273609161377f, 0.6533640027046204f, -0.9062879085540771f, 1.4614392518997192f, 1.6020095348358154f, -0.8768263459205627f, 0.43399056792259216f, -0.06297583878040314f, -0.9222075343132019f, -1.3687634468078613f, 0.5819101333618164f, 1.2366944551467896f, 1.6560721397399902f, 1.2221823930740356f, 1.4343817234039307f, 0.982934296131134f, 0.003909424878656864f, 0.16369947791099548f, 1.25956130027771f, 1.0566132068634033f, -1.0371782779693604f, -1.5509816408157349f, 1.1234275102615356f, 1.1463638544082642f, -0.3711417317390442f, 1.1168509721755981f, -1.9369032382965088f, -0.8143561482429504f, 1.0661178827285767f, 0.771891176700592f, -1.9291707277297974f, 0.034095875918865204f, -0.6951659917831421f, 1.3456140756607056f, 0.34626761078834534f, 0.8155362606048584f, -0.7486876249313354f, -0.7881835699081421f, 0.5057575702667236f, 1.2215378284454346f, -0.9714295268058777f, 0.27692535519599915f, -1.5334482192993164f, 1.146209955215454f, -0.18189188838005066f, -1.2797164916992188f, -0.5713837146759033f, 1.1689343452453613f, -1.8829604387283325f, -0.369537353515625f, -2.0042972564697266f, -0.8240620493888855f, -0.3844030499458313f, 0.5684375166893005f, -1.137097716331482f, -0.31372663378715515f, 0.15282166004180908f, 1.0580254793167114f, -0.17965702712535858f, -1.178733229637146f, -1.2379671335220337f, -1.5433486700057983f, -0.6671661734580994f, 0.9025653600692749f, 0.304572194814682f, -0.47141557931900024f, 0.005512263625860214f, 0.8932762742042542f, 0.14350247383117676f, 0.7622402906417847f, -0.758182168006897f, -0.7234253287315369f, 0.7250988483428955f, -0.05147926136851311f, -0.6577823162078857f, -0.036672789603471756f, 0.514220118522644f, -0.6020243763923645f, -0.09203547239303589f, -1.7167327404022217f, -0.7002271413803101f, 0.4779217541217804f, 0.46033239364624023f, -0.386881560087204f, -0.0812682956457138f, 0.5568539500236511f, 0.04228341206908226f, -0.3627345860004425f, -1.3895257711410522f, 0.17128385603427887f, -1.2317392826080322f, 0.6718097925186157f, -0.15104348957538605f, 0.3213997781276703f, 0.9548746347427368f, 0.04617787152528763f, 1.7811017036437988f, 0.8509474396705627f, 1.179113507270813f, -0.35717684030532837f, 1.1261653900146484f, 1.2983627319335938f, -0.6900910139083862f, 0.4424906373023987f, 1.1874427795410156f, -0.21165163815021515f, -1.2463988065719604f, 0.9242297410964966f, 0.13773603737354279f, -0.060615699738264084f, 0.18482893705368042f, 0.09389957040548325f, -0.2936820983886719f, 0.36884430050849915f, -1.0105520486831665f, -1.1568186283111572f, 0.04061844199895859f, 1.7561308145523071f, -0.5621538162231445f, 0.19418247044086456f, 0.027558235451579094f, -0.1787608563899994f, -1.4996418952941895f, 0.26971569657325745f, 1.2250828742980957f, -1.170371651649475f, -0.11677189916372299f, -1.35635244846344f, 0.5242968797683716f, 1.0622302293777466f, -0.620258629322052f, -1.7659958600997925f, -0.06464217603206635f, 0.21129246056079865f, 0.825295627117157f, -0.4999869465827942f, 2.0010547637939453f, -0.8498311638832092f, -0.5403968691825867f, 0.628537118434906f, -0.7642602920532227f, -2.773613929748535f, -0.22911065816879272f, -1.0603324174880981f, 1.0186315774917603f, 1.763922929763794f, -1.4548780918121338f, -2.5379507541656494f, 3.445625066757202f, 1.4860185384750366f, 2.268347978591919f, -0.5500510334968567f, -0.12460697442293167f, 1.166976809501648f, 1.2409895658493042f, -0.7066188454627991f, 0.9183834791183472f, -0.02982438914477825f, -0.3448553681373596f, 1.0654057264328003f, -0.15129554271697998f, -0.048379626125097275f, -0.5897568464279175f, 0.09694309532642365f, -1.1129159927368164f, 0.5585124492645264f, -1.7732727527618408f, 0.023851748555898666f, -1.156516194343567f, -0.34847185015678406f, -1.6212294101715088f, -0.8558416366577148f, 0.40576910972595215f, -0.91661536693573f, -1.0765177011489868f, 0.5447612404823303f, 1.1705763339996338f, 0.9127081632614136f, -0.6960206627845764f, -0.7912065982818604f, -0.4341987073421478f, 0.3491472005844116f, -0.9907275438308716f, -1.5203826427459717f, 0.9009959697723389f, 0.07418181002140045f, -0.9035959839820862f, 0.9040606021881104f, 0.25104597210884094f, 0.4702746272087097f, -0.5477657318115234f, 1.522364616394043f, 1.6411151885986328f, -1.3906173706054688f, 0.5155009627342224f, 0.37119555473327637f, -0.015211731195449829f, 0.8483667373657227f, 0.634861409664154f, 0.5293999314308167f, -0.18842656910419464f, 1.757521390914917f, 1.3967915773391724f, -0.5659265518188477f, 0.1281542032957077f, 1.8777929544448853f, -0.3289257884025574f, 0.02351505309343338f, 0.428303986787796f, -0.7096118927001953f, 0.723761260509491f, 1.5117958784103394f, 0.5427215099334717f, 2.0354838371276855f, 1.0238486528396606f, -0.39082857966423035f, -1.2726235389709473f, -0.6254484057426453f, 0.9178156852722168f, 0.3840969204902649f, -0.19802889227867126f, 0.5223742127418518f, 0.05841278284788132f, 0.28558799624443054f, -1.0703928470611572f, 0.9307976365089417f, -0.18846826255321503f, -0.8914596438407898f, 0.12508316338062286f, 0.3772087097167969f, 0.9574744701385498f, -1.9291036128997803f, 0.5764585137367249f, 0.20935998857021332f, 0.5120382905006409f, -0.7815778851509094f, -1.478035569190979f, 0.051756951957941055f, 2.1100034713745117f, 0.020343216136097908f, -0.6159464716911316f, -0.4243218004703522f, -0.5821011662483215f, 0.5247467160224915f, -0.8761705756187439f, 0.48434850573539734f, 0.019129564985632896f, 0.03497670218348503f, -0.35594069957733154f, 1.505028486251831f, -0.8405776023864746f, -0.15094131231307983f, -0.5910253524780273f, 0.4260155260562897f, 0.7726138234138489f, 0.9289091229438782f, -1.0194525718688965f, -0.4821736812591553f, -0.4351550042629242f, 0.3517809212207794f, 1.2222594022750854f, -0.030398759990930557f, -1.0700304508209229f, -0.4833625555038452f, -0.5188538432121277f, -1.1346086263656616f, -1.0445209741592407f, 0.4199317395687103f, 0.8357481956481934f, 0.9475129246711731f, -0.34265488386154175f, -0.9238923788070679f, 0.3848164975643158f, 0.4773769974708557f, -1.2878144979476929f, 0.2491273283958435f, 0.32681646943092346f, 1.0925219058990479f, -0.7518913149833679f, -1.2260730266571045f, -0.3927396237850189f, -0.7111839056015015f, 0.5547724366188049f, 1.4059346914291382f, -0.22924941778182983f, 0.4364280700683594f, 1.2370060682296753f, 1.1473842859268188f, 0.29434895515441895f, -0.1789979487657547f, 1.9290111064910889f, 1.1900452375411987f, -0.10487240552902222f, -0.5064011812210083f, 2.4400694370269775f, -0.33963847160339355f, 1.7511601448059082f, 1.3401062488555908f, -0.9492565989494324f, -0.8270853161811829f, -1.2403943538665771f, -0.2053070068359375f, -0.2497664988040924f, -0.7054344415664673f, -1.7715047597885132f, 0.4937354028224945f, -0.17678959667682648f, 1.410936951637268f, 2.476804733276367f, -0.8441710472106934f, -1.912861704826355f, 0.007326157763600349f, 0.19589348137378693f, 1.5108956098556519f, 1.0457415580749512f, 1.560778021812439f, 0.24919238686561584f, -1.4739270210266113f, 0.22397132217884064f, 1.1544770002365112f, 0.4349344074726105f, 0.5612083077430725f, 0.6398148536682129f, 0.7980097532272339f, 0.5769069790840149f, -0.22143854200839996f, 0.31076598167419434f, -0.052180398255586624f, 1.7757917642593384f, -1.198473572731018f, -1.1297553777694702f, -1.0952261686325073f, -1.1415528059005737f, 0.5066304802894592f, -0.6875526905059814f, -1.7462857961654663f, 0.06312361359596252f, -1.7986558675765991f, 0.6912683248519897f, 0.49413225054740906f, 0.14730526506900787f, 2.274590015411377f, -1.4762635231018066f, 1.167490005493164f, -0.18522614240646362f, 0.37665531039237976f, -2.800556182861328f, 1.1855055093765259f, 0.7821402549743652f, 0.2779245674610138f, -0.489844411611557f, 0.7103408575057983f, -0.6894504427909851f, -0.6078459620475769f, 0.5198960900306702f, -0.18495284020900726f, -1.3136197328567505f, -0.783242404460907f, -0.47829392552375793f, 1.730208158493042f, 1.0595130920410156f, -0.251127690076828f, 1.2796905040740967f, 1.1880545616149902f, 0.010817415080964565f, 1.351500153541565f, 0.399687796831131f, -0.10175959765911102f, -0.2829807698726654f, -0.5876336097717285f, -0.7546091079711914f, -1.0380902290344238f, -0.6061143279075623f, -0.26778313517570496f, -0.11727923899888992f, 2.056168794631958f, 1.2978315353393555f, -0.527733564376831f, 0.2199985533952713f, 2.055619716644287f, 0.3816293179988861f, -0.8949641585350037f, 0.6981958150863647f, -0.9663392901420593f, 0.6600029468536377f, -0.3451770842075348f, -0.033411234617233276f, -0.10130416601896286f, 0.6449755430221558f, 1.5684525966644287f, -0.630508303642273f, -0.23472319543361664f, 0.3285580277442932f, -0.47698524594306946f, 0.3249095380306244f, 0.6602651476860046f, -1.652677059173584f, -1.596110463142395f, 0.05211031809449196f, -1.4231536388397217f, -0.16193436086177826f, -0.7287773489952087f, -0.8276495337486267f, -0.9271824359893799f, -0.24771355092525482f, -0.283514142036438f, -1.265464186668396f, -1.345624327659607f, 0.14869070053100586f, -0.5625298023223877f, 1.319007396697998f, -1.1078379154205322f, -0.38139888644218445f, 0.673722505569458f, 0.2882064878940582f, -0.7846477627754211f, 0.886084258556366f, -1.4957531690597534f, -0.3524166941642761f, -0.6002296209335327f, -0.5345580577850342f, -1.4335700273513794f, -0.06124546006321907f, -1.0124340057373047f, 0.7134581804275513f, -0.3809734880924225f, 1.6037938594818115f, -0.6744462847709656f, 0.03632983937859535f, 0.2812545895576477f, 2.2968475818634033f, 1.8312735557556152f, 0.0294801015406847f, 0.7718657851219177f, -0.6380594372749329f, -0.05797451362013817f, -1.4051520824432373f, -0.8756160736083984f, -0.9422666430473328f, -0.14667245745658875f, -1.9506268501281738f, -1.632828712463379f, 0.7152358293533325f, -1.8892930746078491f, 2.4672696590423584f, 0.08700139075517654f, 1.649484395980835f, -0.6159215569496155f, -0.5199353098869324f, 0.01673818565905094f, 0.5336771607398987f, 0.2974875271320343f, 1.8911962509155273f, 0.009384111501276493f, 1.4318883419036865f, -0.49658793210983276f, -0.6031076312065125f, -0.32575365900993347f, 1.9036880731582642f, -1.842443585395813f, -0.16547387838363647f, -0.25706610083580017f, 1.3179066181182861f, -0.6072822213172913f, -0.16537998616695404f, 0.6803566813468933f, 0.1680675745010376f, 1.6327744722366333f, 1.3595441579818726f, 0.6304294466972351f, 0.13153165578842163f, 0.36382463574409485f, -0.7942026853561401f, 2.110002040863037f, -1.1197839975357056f, 0.13227719068527222f, -0.3069077730178833f, 2.2180261611938477f, 0.7556464672088623f, -2.05973744392395f, -0.27732527256011963f, -0.12983323633670807f, -1.0805774927139282f, -1.4954025745391846f, -0.8487699627876282f, 0.8632985949516296f, -0.18222470581531525f, 0.3396982252597809f, 0.023615580052137375f, -0.5582384467124939f, -2.211786985397339f, -0.792870283126831f, 1.4188532829284668f, 1.240154504776001f, 1.2471561431884766f, 1.5289140939712524f, -0.24465647339820862f, 0.09731336683034897f, 0.8652670383453369f, -0.29969948530197144f, -1.841177225112915f, 1.1419644355773926f, 0.3660371005535126f, 1.273138165473938f, -0.7165514230728149f, 0.3878994286060333f, -0.5754631757736206f, 1.2297093868255615f, -0.4566229581832886f, -0.6573424935340881f, 0.7881364822387695f, 0.3378683924674988f, -1.988023042678833f, 0.7956935167312622f, 0.9823195338249207f, -0.4144572615623474f, 1.3572540283203125f, 0.3109254240989685f, -0.6753576993942261f, -1.5371414422988892f, 0.9313072562217712f, -2.2768142223358154f, 0.4319401681423187f, 0.07773400843143463f, -1.5976078510284424f, 1.848445177078247f, 1.5493229627609253f, 0.1915358453989029f, -1.266395926475525f, -2.1607372760772705f, 0.7240417003631592f, -0.7832543253898621f, 1.1483440399169922f, -0.10642467439174652f, -0.32092228531837463f, 1.0402002334594727f, 0.11275313794612885f, 0.37306782603263855f, 0.8337973356246948f, 1.8036106824874878f, 0.7735506892204285f, -1.1966158151626587f, -0.6088662147521973f, 0.16352267563343048f, -0.3072216808795929f, -0.5694004893302917f, 0.13301636278629303f, -1.508280873298645f, -0.8582409024238586f, -0.7953895330429077f, -1.2091906070709229f, -0.7641999125480652f, 0.2154434323310852f, 0.2061154842376709f, 0.3153077960014343f, -0.3388381898403168f, -0.6360014081001282f, -0.45390239357948303f, -2.7026309967041016f, 0.6710167527198792f, 0.8398252725601196f, -2.002260208129883f, -0.6277503967285156f, -1.3664311170578003f, -0.47482022643089294f, -1.1213213205337524f, 0.07767737656831741f, -0.44291654229164124f, -1.2841796875f, -1.0159707069396973f, 1.3361929655075073f, -0.46696820855140686f, -0.2509534955024719f, 1.4244478940963745f, -0.6108700037002563f, -0.4096096158027649f, -0.4688802659511566f, -1.2304245233535767f, -0.14591790735721588f, 0.9639950394630432f, -0.9581076502799988f, 0.8579769134521484f, 0.17619068920612335f, -0.009090420790016651f, 2.2291040420532227f, -0.599851667881012f, 0.5864706635475159f, -0.40187644958496094f, 0.7005379796028137f, 2.269195318222046f, 1.1897660493850708f, -0.530225396156311f, 0.22658583521842957f, 0.8770406246185303f, -0.42359691858291626f, 0.04045804589986801f, -0.3348093032836914f, -1.6933249235153198f, 0.578863799571991f, 0.253021240234375f, 0.8249357342720032f, 0.17797666788101196f, 0.21032211184501648f, -1.3109723329544067f, 1.4387873411178589f, 1.3104835748672485f, -0.4802001118659973f, 0.2532861530780792f, 0.34186819195747375f, -2.0921499729156494f, -1.4881683588027954f, 0.24693161249160767f, -1.7462626695632935f, 0.28935855627059937f, 1.5511679649353027f, 0.8895691633224487f, 0.4553866684436798f, -1.143309235572815f, -0.06552600860595703f, 0.03079218976199627f, -1.0684497356414795f, -0.3178943395614624f, 2.2536263465881348f, -0.19900000095367432f, 0.5933657884597778f, 1.5937228202819824f, 0.8582064509391785f, 0.11251302808523178f, 0.2912493050098419f, 0.12979160249233246f, 0.08444352447986603f, 2.015190601348877f, 0.5599141120910645f, 1.3394620418548584f, 0.6110420823097229f, -0.5921895503997803f, -0.16634242236614227f, -0.3773656487464905f, 1.1717573404312134f, 0.6101416945457458f, 1.917343020439148f, 2.563725471496582f, 3.035121202468872f, -0.534939169883728f, -0.4571112394332886f, 0.802577018737793f, -0.28734639286994934f, 0.12288182973861694f, 0.21209175884723663f, -0.996752142906189f, 0.6648997068405151f, -1.1771180629730225f, 0.517611563205719f, 2.260444164276123f, 0.8793008327484131f, -1.4391361474990845f, -0.47865501046180725f, -0.12678508460521698f, -1.1487538814544678f, 0.9171100854873657f, 0.9602430462837219f, 0.20192822813987732f, -0.6621134877204895f, 0.283156156539917f, 1.1861251592636108f, -0.6021904945373535f, -0.2499617040157318f, 1.7409440279006958f, -0.7325262427330017f, -0.3309515118598938f, -0.7796709537506104f, 1.6620672941207886f, -0.0578240342438221f, 0.2314254492521286f, 0.22709836065769196f, 2.065865993499756f, 0.4171837866306305f, 0.3846955895423889f, -0.01396828331053257f, 0.5790372490882874f, 0.2881997525691986f, 0.8778835535049438f, 0.5851041674613953f, -0.29608187079429626f, 0.33585336804389954f, -0.7194162011146545f, -0.07809803634881973f, 0.35557520389556885f, -1.7239394187927246f, 0.7924410104751587f, 0.030582552775740623f, 1.0178154706954956f, 1.6657739877700806f, 0.17066167294979095f, -0.2777872681617737f, -0.19989891350269318f, 1.6998308897018433f, 0.7318398952484131f, 1.2774115800857544f, -0.3473694622516632f, -0.7603669166564941f, 1.2199066877365112f, -0.3723330795764923f, -1.8119957447052002f, -0.9909027814865112f, -0.3076462149620056f, 0.01513738464564085f, -1.9378867149353027f, -1.8149548768997192f, 1.3613444566726685f, -0.7856798768043518f, -2.4690840244293213f, 2.432499408721924f, 0.27381452918052673f, 0.8801257610321045f, -0.49671268463134766f, 0.056602153927087784f, 1.435631513595581f, 0.36377331614494324f, 0.46456536650657654f, 1.9551998376846313f, 0.6760215759277344f, 1.1773208379745483f, -0.5477144718170166f, -0.48883718252182007f, -0.2574329078197479f, 0.46062397956848145f, -1.6101100444793701f, 0.2585121691226959f, 1.7188366651535034f, 0.5200220942497253f, -1.301006555557251f, -1.5038570165634155f, -0.31398269534111023f, 1.2563191652297974f, -0.5480050444602966f, -0.06532658636569977f, 2.6805763244628906f, -0.9649611115455627f, -0.18265697360038757f, 0.3190588653087616f, 0.006673657335340977f, 0.566218376159668f, 1.7436981201171875f, 1.3330131769180298f, 1.3450311422348022f, -1.0255969762802124f, 1.3098556995391846f, -0.4485261142253876f, 0.8979019522666931f, -0.1220834031701088f, -1.0596239566802979f, 0.14630116522312164f, -0.870776891708374f, -0.24674776196479797f, -1.2373387813568115f, 0.8207234740257263f, -0.5435441136360168f, 1.1541569232940674f, -0.38771313428878784f, 0.12649448215961456f, -1.2653104066848755f, 1.7627143859863281f, -0.2666304111480713f, 0.5257315635681152f, 0.6358874440193176f, 0.10120877623558044f, 0.17400647699832916f, 1.1357200145721436f, 0.036105670034885406f, -0.6797627210617065f, -0.16857317090034485f, 0.26855215430259705f, -1.4135205745697021f, -2.3971786499023438f, 0.7723278403282166f, -0.15770527720451355f, 0.45117413997650146f, -2.815762519836426f, 0.19697873294353485f, 0.2619018256664276f, -0.8475749492645264f, 0.47121289372444153f, 0.3821605145931244f, -0.2688539922237396f, 1.0989755392074585f, -1.0098387002944946f, 0.688886284828186f, 0.5349103808403015f, 2.787760019302368f, 0.38038408756256104f, 0.22671592235565186f, 0.5783246755599976f, 0.6597211360931396f, -0.28375279903411865f, -0.6992143392562866f, 0.7167068123817444f, -0.09313177317380905f, 0.6840168237686157f, -0.19578158855438232f, -0.9126741886138916f, -0.2800213098526001f, -0.3882693350315094f, 0.10380307585000992f, -1.9663856029510498f, -1.013744592666626f, -1.0242971181869507f, 0.8466978073120117f, 1.2233283519744873f, -0.6524525284767151f, -0.6867133378982544f, 1.1395643949508667f, -0.696540355682373f, -0.39359593391418457f, 0.5099198222160339f, -0.15132363140583038f, 0.6866339445114136f, 0.5310931205749512f, -1.3795819282531738f, -0.6988955140113831f, 0.3475550413131714f, 1.001903772354126f, 0.7994512319564819f, 0.8503185510635376f, 0.791765034198761f, -0.8879274725914001f, 1.02428138256073f, 0.1911749392747879f, 0.8436497449874878f, -0.02484050951898098f, -0.7801733613014221f, 0.625577986240387f, 1.5644149780273438f, 0.5320438146591187f, 1.0700255632400513f, -0.5013832449913025f, -0.3839770257472992f, 0.8672234416007996f, 0.8700053095817566f, 1.8044092655181885f, 0.5074418187141418f, -1.0803741216659546f, 1.7308460474014282f, -0.009462441317737103f, 1.9249420166015625f, -0.3632628321647644f, 0.6629142165184021f, -0.6219039559364319f, 1.013214111328125f, -1.5853021144866943f, -0.9035424590110779f, -0.1714278757572174f, 0.9996393918991089f, 0.2449631243944168f, 0.07007400691509247f, 1.4646685123443604f, 0.4391697347164154f, 1.424518346786499f, -0.32468941807746887f, 0.35460788011550903f, -0.3405316174030304f, -0.6941221952438354f, 0.6556671857833862f, -1.087321400642395f, -1.1486494541168213f, 2.4220407009124756f, 0.7684311866760254f, -1.5410068035125732f, -0.411981463432312f, 1.3056882619857788f, -0.338508278131485f, 0.2913444936275482f, -0.8357943892478943f, -1.1213253736495972f, 0.3143851161003113f, -0.42382940649986267f, -0.43293496966362f, -0.9816297292709351f, 0.058465536683797836f, -1.3252078294754028f, -0.7915613055229187f, 0.47723037004470825f, 0.43918681144714355f, -0.3302071988582611f, 1.7920981645584106f, -1.0610337257385254f, -0.09547755867242813f, -1.1624996662139893f, -0.6900646686553955f, 0.3918372392654419f, -0.0669134333729744f, 1.0711755752563477f, 1.3083903789520264f, -0.05556102097034454f, 0.7882350087165833f, 0.37722504138946533f, -0.3213580846786499f, 0.5956750512123108f, -0.5032515525817871f, -0.7394251227378845f, -0.7672699093818665f, -0.040719106793403625f, 0.15840405225753784f, -0.4783642590045929f, 0.4646506905555725f, -0.49036842584609985f, -0.6546841263771057f, 2.7124791145324707f, 0.4293099343776703f, -1.0469098091125488f, -1.0857888460159302f, -0.05841507390141487f, 0.5456286668777466f, 0.2792662978172302f, 2.1168277263641357f, -1.5103271007537842f, -1.307977557182312f, -1.9681334495544434f, -0.8398545384407043f, 1.3870203495025635f, 0.9826740026473999f, -0.2890629470348358f, 1.0335131883621216f, -0.19352838397026062f, 0.07122802734375f, -0.16152557730674744f, 1.0519514083862305f, -1.4766151905059814f, -1.2671267986297607f, 0.2393292933702469f, 1.2190254926681519f, 0.17837846279144287f, -1.3159326314926147f, 0.646186888217926f, 1.6116095781326294f, -0.05114508792757988f, 0.41359904408454895f, -0.20463405549526215f, -2.1418421268463135f, 1.7502615451812744f, -1.4018466472625732f, -0.5717736482620239f, -0.38931718468666077f, -0.9860185384750366f, 0.5779818296432495f, 0.4738202393054962f, -0.7853313088417053f, -0.1793491244316101f, -0.1285407692193985f, 0.9474418759346008f, -0.07980115711688995f, -0.6013848185539246f, -0.5239112377166748f, -0.1949448138475418f, 0.7724379897117615f, -0.11171314865350723f, 0.5611289739608765f, -1.5560880899429321f, 1.4754453897476196f, 1.4865750074386597f, -0.02104124240577221f, 0.031055578961968422f, 1.0900859832763672f, -0.5583686828613281f, 1.3067461252212524f, -0.9643430113792419f, 1.6326349973678589f, -1.144444465637207f, 0.13422854244709015f, 1.668276309967041f, -0.6358500719070435f, -0.8220173120498657f, 1.151316523551941f, -0.9632699489593506f, -0.17086558043956757f, 0.14713206887245178f, -0.13801060616970062f, -0.1489359736442566f, -0.06645472347736359f, -0.880412757396698f, 0.1913401335477829f, -0.05544678866863251f, -0.2204407900571823f, -1.1233654022216797f, -1.1245696544647217f, 0.674127995967865f, 0.2520127594470978f, 0.7975156307220459f, 0.698857843875885f, -0.4366423487663269f, -2.0883853435516357f, -1.366032361984253f, 0.09937097132205963f, -0.3652037978172302f, 1.2572640180587769f, -0.8791263699531555f, 1.3767470121383667f, 0.6487414836883545f, -0.22251465916633606f, -0.7945832014083862f, 1.3094075918197632f, 1.1708104610443115f, 0.9832903742790222f, -0.7685240507125854f, -0.5897563099861145f, -0.008548072539269924f, 0.7963455319404602f, -0.6708536148071289f, -0.16284485161304474f, -1.8156386613845825f, 0.1582425981760025f, 1.5490630865097046f, 0.6892044544219971f, -1.5244767665863037f, 0.1314752995967865f, -0.5694751739501953f, -0.07285541296005249f, 0.02161409705877304f, 0.27252840995788574f, 1.5375657081604004f, -0.16460900008678436f, 0.04459971562027931f, 0.4962165355682373f, 0.9521092176437378f, 0.18977606296539307f, 1.1129406690597534f, -1.742975115776062f, 0.015372429974377155f, 0.43921586871147156f, -0.4629494845867157f, 0.683012068271637f, 0.0549420528113842f, 1.1227099895477295f, -0.9156265258789062f, 0.7202309370040894f, -1.7771120071411133f, -0.49309614300727844f, -0.3537367880344391f, 0.6029451489448547f, 1.4749457836151123f, -1.2571945190429688f, 1.1716068983078003f, -1.2938947677612305f, -0.027610722929239273f, -0.04813985526561737f, -1.6245776414871216f, -0.1380276381969452f, -0.7646363377571106f, -0.24534782767295837f, 0.21846671402454376f, 0.8769089579582214f, -1.0646121501922607f, 0.50408536195755f, 0.15752233564853668f, -0.5226418375968933f, -1.4756296873092651f, 0.25705486536026f, -1.7179490327835083f, 1.3074569702148438f, -0.4480331838130951f, 1.1221709251403809f, 1.4023377895355225f, 1.4818451404571533f, -1.4184188842773438f, 0.2722645699977875f, 1.2556324005126953f, -1.6670103073120117f, 1.050782322883606f, 0.13771948218345642f, -1.1567072868347168f, 1.0360978841781616f, -0.8660301566123962f, -1.0625808238983154f, 1.0239890813827515f, 0.7085686922073364f, 1.8764188289642334f, 2.794304609298706f, 0.38867735862731934f, 1.6592570543289185f, -0.8973153829574585f, -0.428587943315506f, -0.8077448010444641f, -2.2226719856262207f, 1.3841030597686768f, 0.27507975697517395f, 1.8155642747879028f, 0.5318379998207092f, 1.278135895729065f, -0.632580041885376f, -1.036582350730896f, 0.294900506734848f, -0.8918796181678772f, -0.08464081585407257f, 0.7254555821418762f, 0.40998774766921997f, -0.0699654296040535f, 0.8239985108375549f, 2.0619850158691406f, -1.0847980976104736f, -1.5027097463607788f, 0.4682971239089966f, -2.0921216011047363f, -0.31479185819625854f, 0.35259687900543213f, -0.6293240189552307f, -1.997800350189209f, -0.6937687397003174f, 0.43931636214256287f, -0.2434539496898651f, -1.389511227607727f, -0.0009331775945611298f, 0.4944293797016144f, -1.3989880084991455f, 0.3048802316188812f, 0.612401008605957f, -1.0865437984466553f, -0.7029951810836792f, -0.651735246181488f, 0.02101830020546913f, -0.07500791549682617f, -1.676796793937683f, 1.5087897777557373f, -0.8013211488723755f, -1.030982494354248f, -1.4118940830230713f, -0.8552696704864502f, -0.1083751767873764f, 2.1871159076690674f, -0.6815629601478577f, -1.4610782861709595f, -0.5785004496574402f, 2.1495704650878906f, -0.179605171084404f, 1.1426492929458618f, -0.054547298699617386f, 0.40587425231933594f, 0.6724022030830383f, 0.19652637839317322f, -0.07756809890270233f, -1.4263975620269775f, -1.0503880977630615f, -1.5335310697555542f, 0.4794066250324249f, 1.1617707014083862f, 0.8538435697555542f, -0.20266015827655792f, -0.725740909576416f, -0.9262224435806274f, 0.8973854184150696f, -0.7538284063339233f, -0.8116031885147095f, 0.5351168513298035f, 1.988852858543396f, 1.0685486793518066f, -0.5014570951461792f, 0.7195857763290405f, 1.110618233680725f, 0.1288852095603943f, -1.2922874689102173f, 0.6346917748451233f, 1.5678290128707886f, 0.03146770969033241f, 1.8645329475402832f, -0.8617974519729614f, 0.051693860441446304f, -1.4663174152374268f, -0.26106584072113037f, -0.06878392398357391f, 0.8156912922859192f, 0.4850883483886719f, -2.2269856929779053f, 0.5959717631340027f, 0.21369199454784393f, 0.2891088128089905f, 0.3474847674369812f, -0.43252453207969666f, -0.5997704267501831f, 0.2910449504852295f, -0.8439115881919861f, -0.013285640627145767f, -0.5315346121788025f, 0.08023621141910553f, -0.6925831437110901f, -0.6154842972755432f, -1.4682725667953491f, 0.005962289869785309f, -0.17264695465564728f, -1.3461240530014038f, 0.5067698955535889f, -0.7377328872680664f, -0.006388451438397169f, -1.411285400390625f, 0.23984402418136597f, 0.4191838204860687f, 1.395574688911438f, 0.976121723651886f, 0.40693262219429016f, -0.6316997408866882f, 1.5522570610046387f, 0.2695809602737427f, 1.6630321741104126f, 1.0006709098815918f, -0.6625555157661438f, -0.05369861051440239f, 0.2828640341758728f, 0.8807763457298279f, -0.3764995038509369f, 1.8677223920822144f, 2.1043002605438232f, -0.8154667019844055f, -0.4016047418117523f, -0.07730520516633987f, 0.4082213044166565f, -0.7410086393356323f, -2.3087081909179688f, -0.031602293252944946f, -1.454459547996521f, 0.7048721313476562f, -0.5495360493659973f, -1.0822854042053223f, -1.1814165115356445f, -0.7662639021873474f, 2.26790452003479f, -0.9821573495864868f, -0.7700114250183105f, -0.6950612664222717f, -0.47602152824401855f, -2.168846368789673f, -0.4960733652114868f, 1.8063979148864746f, -2.195838451385498f, -1.2756946086883545f, -0.23146040737628937f, -0.6978404521942139f, 0.05868193879723549f, 0.3181943893432617f, -0.9001671671867371f, 1.4585676193237305f, 0.9991294741630554f, -0.958099365234375f, -1.5130436420440674f, 0.8987486362457275f, 0.6024074554443359f, 1.213675856590271f, -0.9290968179702759f, 0.9378061294555664f, 0.32025381922721863f, -0.6372569799423218f, 0.5549997091293335f, 0.4728609323501587f, 1.5382329225540161f, -2.230900526046753f, 2.1022186279296875f, 1.7723872661590576f, -0.29818224906921387f, 1.774613618850708f, -0.403571754693985f, -0.763476550579071f, -0.13895398378372192f, -1.8085858821868896f, -0.1992855966091156f, -0.38452786207199097f, 0.772721529006958f, -0.761553168296814f, 0.32450324296951294f, -0.6566532254219055f, 1.0444575548171997f, -1.0310343503952026f, 0.13313262164592743f, 0.32965973019599915f, -0.7564722895622253f, 1.5161906480789185f, -0.442477822303772f, -0.19663105905056f, -0.5199072957038879f, 0.19425715506076813f, 0.468250036239624f, 0.658501386642456f, -0.16666415333747864f, -0.4670298099517822f, 1.4936802387237549f, -0.8678151965141296f, -0.26301294565200806f, 0.20588766038417816f, 0.022425031289458275f, 0.6587340831756592f, -0.20476730167865753f, -1.0465848445892334f, -1.2654074430465698f, -0.3633078634738922f, -0.4297706186771393f, 0.6679531335830688f, 1.0863715410232544f, 0.7616772055625916f, -0.011849306523799896f, -1.2028323411941528f, 0.5853366255760193f, -0.10431576520204544f, 0.2191448211669922f, -0.8329556584358215f, -0.2799490988254547f, 0.6428387761116028f, 1.405016303062439f, 1.3759163618087769f, 1.4840714931488037f, -1.5530608892440796f, -0.9330264925956726f, -1.1588698625564575f, -0.48916497826576233f, -0.6450856328010559f, -0.11284743994474411f, -2.4587764739990234f, 0.7599589824676514f, 0.9755558371543884f, 0.05122251436114311f, 0.960921049118042f, -0.6847841143608093f, 0.40178006887435913f, -0.9434769749641418f, -0.23219886422157288f, -0.2043316662311554f, -0.8605437278747559f, -0.3255579173564911f, -0.7162472605705261f, -0.08609210699796677f, -3.6307897567749023f, -0.6838181614875793f, -0.489888072013855f, -1.0136138200759888f, -0.8829219937324524f, 1.1272135972976685f, -1.2231221199035645f, -0.7950332760810852f, 0.6623095273971558f, 2.028413772583008f, 0.8947616219520569f, -1.1885942220687866f, -0.400406152009964f, 0.9303890466690063f, -1.0271012783050537f, 0.60735023021698f, -2.186370372772217f, -1.2513858079910278f, -1.5937029123306274f, -1.391968846321106f, -0.7062578201293945f, 0.5445007681846619f, 0.0782814472913742f, -1.7333787679672241f, -0.8157885074615479f, 0.3931215703487396f, -0.11953603476285934f, 0.14065788686275482f, 1.6809849739074707f, -0.2840443551540375f, -1.4785282611846924f, 0.227834552526474f, 0.26440107822418213f, -0.0759536400437355f, 0.9481480121612549f, 0.8856539726257324f, -1.2799997329711914f, -0.2185828685760498f, -0.15562321245670319f, 1.1858826875686646f, 1.7509973049163818f, 0.4133126139640808f, 0.7854304909706116f, 0.64668208360672f, -0.11213088780641556f, 0.8463886380195618f, 0.045808374881744385f, -0.6186387538909912f, -0.5598823428153992f, 0.3789231479167938f, -0.4264816343784332f, 0.9161636829376221f, 0.13592901825904846f, 0.4121098220348358f, 0.40913498401641846f, 0.4465799629688263f, -0.07119281589984894f, 0.4638987183570862f, -0.5617773532867432f, -0.23070071637630463f, -0.5451535582542419f, 0.018565375357866287f, -0.10694064944982529f, 1.1308897733688354f, 1.4536970853805542f, 0.6201399564743042f, -1.4827039241790771f, 0.4850791096687317f, -0.28107115626335144f, -1.132474660873413f, 0.4989188015460968f, 1.1301037073135376f, -0.3190559148788452f, -0.02982708252966404f, -0.4788385033607483f, 0.3957928717136383f, 0.5253366231918335f, -1.6755516529083252f, 0.35308536887168884f, -0.12078773975372314f, -0.2962888479232788f, -0.08987566083669662f, -0.45044565200805664f, -1.0356383323669434f, 1.7253460884094238f, -2.3891191482543945f, -1.0028913021087646f, -0.3333589434623718f, 0.9801297783851624f, 1.2715272903442383f, -0.7523331046104431f, -0.13584411144256592f, 0.6945744156837463f, -1.9437434673309326f, 0.3301672339439392f, 1.6046744585037231f, -0.4702155888080597f, 1.209662675857544f, 0.8889638781547546f, 0.1620996594429016f, 0.12699493765830994f, 0.7177993655204773f, 0.06786083430051804f, 0.8935588598251343f, 0.11956664174795151f, 1.0848652124404907f, -0.47686854004859924f, 0.2491820603609085f, 0.730522096157074f, 0.09865526109933853f, -0.5309102535247803f, 0.08605138957500458f, -1.499106526374817f, 0.8767158389091492f, -1.1525547504425049f, -0.34564009308815f, -0.8810292482376099f, -1.5831094980239868f, -0.9634820222854614f, -1.0025994777679443f, 0.07062987983226776f, 0.11008060723543167f, 0.7735913395881653f, -0.0040825954638421535f, 0.8591998815536499f, -0.7495615482330322f, -0.38007304072380066f, -1.8807623386383057f, 0.10670913010835648f, 1.1232846975326538f, -0.15919449925422668f, -0.561027467250824f, 0.30757027864456177f, -1.5486440658569336f, -1.054305911064148f, -0.8691359758377075f, -0.06806328892707825f, -0.4993351697921753f, 0.1407848447561264f, 0.07298417389392853f, 1.068840742111206f, -1.1345824003219604f, -0.2619936764240265f, 1.4751683473587036f, 0.1493469625711441f, -0.7424628138542175f, 0.0515340156853199f, -0.4829641580581665f, -0.12765848636627197f, 0.048511698842048645f, -0.6109879612922668f, 1.034902811050415f, 1.2692794799804688f, 1.4105430841445923f, -0.2778305411338806f, -0.2591007351875305f, 1.035352349281311f, 0.4310223162174225f, -0.5226048827171326f, 1.6362823247909546f, 2.1157736778259277f, 0.22881479561328888f, 0.6379362940788269f, 0.4769587814807892f, 0.036591242998838425f, -0.7712311148643494f, 0.11032737791538239f, 1.141363263130188f, 2.2910208702087402f, 0.9097362756729126f, 0.015577469952404499f, 0.7960927486419678f, -1.0888819694519043f, -0.22314855456352234f, -0.42274436354637146f, -0.9645407795906067f, 0.26433172821998596f, -0.4537980556488037f, 1.0802478790283203f, 0.17047974467277527f, 0.5164321660995483f, 0.6420630812644958f, 0.12355764210224152f, -0.6113547086715698f, -0.07968119531869888f, 0.8360456228256226f, -0.9061261415481567f, 0.5791953802108765f, -0.9833644032478333f, 0.04046724736690521f, 0.0772174745798111f, 1.1407610177993774f, -1.1166253089904785f, 0.797391414642334f, -1.2114757299423218f, 0.845003604888916f, 0.917169451713562f, 0.6150916218757629f, -1.4388847351074219f, 0.5669911503791809f, -2.142704963684082f, -0.45466870069503784f, -0.6032977104187012f, -0.13450507819652557f, 0.18595486879348755f, 0.3553427457809448f, 1.6049801111221313f, 0.41566601395606995f, 1.1401103734970093f, 1.4051463603973389f, 0.6006536483764648f, 1.797692060470581f, -0.5229107737541199f, -0.20334042608737946f, -0.45935776829719543f, -0.15298272669315338f, 1.5941132307052612f, -0.1434730738401413f, 0.052168138325214386f, -1.3556065559387207f, 1.5627477169036865f, 1.9382938146591187f, 1.2685400247573853f, 2.026231527328491f, -1.1499669551849365f, -0.8773688077926636f, 0.07689109444618225f, 1.4310468435287476f, -0.18800176680088043f, -1.6620466709136963f, 0.719353199005127f, -0.291658490896225f, 2.5384137630462646f, 0.10889503359794617f, -0.5992977023124695f, -0.394126832485199f, -1.3852146863937378f, -1.254912257194519f, 0.1908181607723236f, -1.0126259326934814f, 0.722631573677063f, -0.34968942403793335f, -0.8847613334655762f, -0.984367311000824f, -0.9932076334953308f, 0.8806517720222473f, -0.09622633457183838f, -1.206941843032837f, -0.49620160460472107f, 0.6914839744567871f, -3.028942346572876f, 1.3682804107666016f, -0.4541301131248474f, 0.45961233973503113f, 0.36240431666374207f, 0.34856998920440674f, -0.821986973285675f, -2.4602458477020264f, 0.37095823884010315f, 0.0116643775254488f, -0.5293033719062805f, -1.6280213594436646f, -0.6806969046592712f, -0.1684180051088333f, 1.5591261386871338f, 0.43387478590011597f, 0.26209768652915955f, 1.2855982780456543f, 0.6979333758354187f, 1.5001028776168823f, -1.1095906496047974f, 0.581849217414856f, 1.3310784101486206f, 1.2475563287734985f, 1.4747735261917114f, 0.3026885390281677f, -1.3104248046875f, -0.5138301253318787f, 0.7339244484901428f, -1.0213230848312378f, -1.202392816543579f, 0.16470061242580414f, 2.572078227996826f, -1.3335460424423218f, 0.546902596950531f, 0.38991880416870117f, 0.15686734020709991f, -0.39347609877586365f, -0.30205461382865906f, 2.149627923965454f, -1.751050591468811f, -0.7567965984344482f, -1.4173346757888794f, 0.5380285382270813f, 0.0939386859536171f, 0.45474377274513245f, -0.6351379156112671f, 0.9299246072769165f, -0.4753718972206116f, 0.9432480931282043f, 0.5524624586105347f, 0.40374335646629333f, 0.6184999346733093f, 0.4535541534423828f, -1.0948821306228638f, -0.320241779088974f, -0.028895698487758636f, 0.7330871224403381f, -1.6074398756027222f, -1.227219581604004f, 1.0834808349609375f, 0.05628223717212677f, 0.20571336150169373f, 0.43112990260124207f, 0.3616485893726349f, -1.1379897594451904f, 0.2402925342321396f, -0.47906389832496643f, 1.4835309982299805f, -1.1785883903503418f, 0.5978415012359619f, 0.5928131341934204f, -0.6638352870941162f, -1.003272533416748f, -0.1671929657459259f, -1.1786675453186035f, 0.8089823126792908f, 0.1908469796180725f, 1.2084790468215942f, -2.24063777923584f, -0.5793269276618958f, -1.1207023859024048f, -1.1295276880264282f, -0.5763978362083435f, -0.26363199949264526f, 0.789819061756134f, -0.4906677007675171f, 0.371157705783844f, -1.484498143196106f, 0.2109839767217636f, -0.8684839606285095f, -1.171597957611084f, -0.9773237109184265f, -0.002813637489452958f, -1.2975432872772217f, 0.3479395806789398f, -0.7253870964050293f, -0.6621626615524292f, -0.2912011444568634f, -2.3239030838012695f, 0.918861985206604f, 1.1420683860778809f, -0.1961705982685089f, -1.5105466842651367f, 0.0809166356921196f, 1.4640601873397827f, -0.4027494490146637f, 1.6002179384231567f, -0.26083904504776f, -0.6423792839050293f, -1.7842353582382202f, -1.4978502988815308f, 1.380509853363037f, 1.877629041671753f, -1.4456785917282104f, -0.28545263409614563f, 0.15021729469299316f, 0.5569579005241394f, -1.6427204608917236f, -0.6389901041984558f, -0.5466802716255188f, -1.0836117267608643f, -1.5072098970413208f, -0.3856680691242218f, 0.9019067287445068f, -0.2348310351371765f, -1.738688588142395f, 0.28999093174934387f, -0.1533069908618927f, -0.9047854542732239f, 0.3279983699321747f, -0.5037038922309875f, -1.9761329889297485f, 0.12128926813602448f, -0.06794015318155289f, 1.1322118043899536f, -1.0514512062072754f, -0.672996997833252f, 0.549104630947113f, -0.32529518008232117f, 0.3177003860473633f, 0.1834784746170044f, 1.7332375049591064f, -0.2832452356815338f, -1.384834885597229f, -0.010661765933036804f, -0.22356857359409332f, 0.5824847221374512f, -0.6592251062393188f, 0.44558772444725037f, -2.2453958988189697f, 0.42556077241897583f, -0.6481807231903076f, -0.25188785791397095f, -1.27206289768219f, 1.1171956062316895f, 1.5053801536560059f, 0.8270988464355469f, 0.7335371971130371f, -0.7486765384674072f, 0.45417457818984985f, -0.3060213625431061f, 0.6338478326797485f, -2.674952745437622f, -0.6763451099395752f, -0.9770270586013794f, 1.5783367156982422f, -0.2843257188796997f, -1.3570187091827393f, -2.2979846000671387f, -0.8282976150512695f, -1.328344464302063f, -0.0004540873342193663f, 0.6817458271980286f, 0.9938647150993347f, -1.4253593683242798f, 0.3879549503326416f, -2.0311243534088135f, 1.2712687253952026f, 0.185286283493042f, 0.07715956121683121f, -0.584597110748291f, -0.5161778330802917f, 0.3677023947238922f, -1.3314623832702637f, 1.5407487154006958f, -0.3439099192619324f, -1.2492866516113281f, -0.8399935364723206f, 0.40630635619163513f, 0.00801965780556202f, -0.3249308466911316f, -0.6664285659790039f, 0.28344160318374634f, 0.5257611274719238f, -1.3125312328338623f, -0.3971971273422241f, -1.5499167442321777f, -0.19803878664970398f, 1.0783737897872925f, 0.5109858512878418f, 0.45442286133766174f, -0.7324386835098267f, -0.1001700609922409f, -0.9963456988334656f, 1.706158995628357f, -0.28345587849617004f, 0.24911366403102875f, -0.5420844554901123f, 0.38604170083999634f, -0.08859143406152725f, -0.7755544781684875f, -0.3989383280277252f, 0.3021530210971832f, 0.8383589386940002f, 0.36416593194007874f, -1.461991786956787f, 0.15854018926620483f, -0.5639234185218811f, 1.251044750213623f, 1.9695827960968018f, 1.165942907333374f, -1.0710505247116089f, -1.4054443836212158f, 2.9302103519439697f, -0.23575513064861298f, 0.08357129991054535f, -0.09462077915668488f, -1.8668535947799683f, -0.34727850556373596f, -0.35426023602485657f, 1.4054851531982422f, 1.4522364139556885f, -0.08649589866399765f, 0.04004557058215141f, -0.10994749516248703f, -0.6241140961647034f, -0.24008060991764069f, -0.39534568786621094f, 0.007234930992126465f, 0.03980718180537224f, -1.2252846956253052f, -0.5507245063781738f, -1.171595811843872f, 1.0313161611557007f, -0.20263050496578217f, 1.3937491178512573f, 1.0930505990982056f, 1.4651625156402588f, 0.2634923756122589f, -0.4779359698295593f, -1.4054735898971558f, 0.7812330722808838f, 0.22484587132930756f, 0.7241055965423584f, -0.6603827476501465f, -0.6927292346954346f, -0.7928091883659363f, -0.4532349705696106f, 0.5121238231658936f, -0.3005983531475067f, -0.44323164224624634f, -0.3902406692504883f, -1.0369137525558472f, -0.19362880289554596f, 0.49349725246429443f, -0.43515950441360474f, -0.6069482564926147f, -1.473711609840393f, -2.3762154579162598f, 1.134980320930481f, -0.9291712641716003f, -0.62174391746521f, 0.47012677788734436f, 1.0591752529144287f, 0.3450745940208435f, -0.04975222051143646f, 1.2706385850906372f, 0.950104296207428f, 0.9791541695594788f, -0.0728548914194107f, 1.3175126314163208f, 0.5881410837173462f, 0.9367974996566772f, 0.9128003120422363f, 0.4015538990497589f, 0.1372641772031784f, 1.041069507598877f, 0.7456277012825012f, 0.0962943509221077f, -1.7508755922317505f, -1.171151041984558f, 1.6682186126708984f, -0.6773730516433716f, -2.147958517074585f, 0.4557051658630371f, 0.061060722917318344f, 0.4539925456047058f, -1.1904879808425903f, 0.2646864056587219f, -0.8139405250549316f, -2.294616460800171f, -0.28324073553085327f, 1.7303272485733032f, -0.46407395601272583f, 1.7161751985549927f, 0.8312178254127502f, 0.2559703290462494f, 0.7086811661720276f, 0.6478610038757324f, -0.06174112856388092f, 0.2679769694805145f, -0.6163668632507324f, -0.4823808968067169f, 0.3113662600517273f, -1.5075656175613403f, -0.32805025577545166f, 0.9543997049331665f, 0.8449776768684387f, -0.8963518738746643f, 0.354059636592865f, -0.15964199602603912f, -0.20036070048809052f, 0.4515315592288971f, 0.987306535243988f, -0.9755827188491821f, 0.2618931531906128f, -2.5518243312835693f, 1.015307068824768f, 2.9806666374206543f, -0.5756134390830994f, -0.42303135991096497f, -1.6033560037612915f, -0.3883368968963623f, -1.6651710271835327f, -1.7486891746520996f, 0.5733898282051086f, -0.17000924050807953f, -1.2242252826690674f, -0.7774103879928589f, -0.662408173084259f, 1.8391462564468384f, 0.4153856635093689f, 0.07132068276405334f, 0.595319926738739f, -0.8092124462127686f, 0.3273322880268097f, -0.4520135819911957f, 0.15657921135425568f, 2.195984125137329f, 0.9013379216194153f, 0.23789522051811218f, -1.9253568649291992f, -0.8211831450462341f, -0.16658450663089752f, -2.5792624950408936f, 1.1836506128311157f, -0.0034865080378949642f, -0.5498691201210022f, 0.3581060767173767f, 0.7652292251586914f, -0.7733491659164429f, 0.6297365427017212f, -0.6815900802612305f, 1.240025281906128f, 0.8412518501281738f, 0.29200538992881775f, -1.805619716644287f, 1.3460670709609985f, 0.02373058721423149f, -0.6370474696159363f, 1.3327691555023193f, -1.63896906375885f, 0.4103865623474121f, 0.25587576627731323f, 1.880447506904602f, 1.9977002143859863f, -1.038170576095581f, 1.879461407661438f, -0.9597489833831787f, -1.338854193687439f, -1.4890133142471313f, 0.9830695986747742f, -0.41451194882392883f, 1.213753581047058f, 0.5297332406044006f, -0.8235613107681274f, 0.3100223243236542f, 0.7781904339790344f, -0.7196370363235474f, 0.8093208074569702f, -0.7462528347969055f, 0.3914867341518402f, 1.1797000169754028f, -2.474013328552246f, 0.43964120745658875f, -0.10444001853466034f, 0.5850116610527039f, -1.6503483057022095f, -1.7112841606140137f, -0.08656994998455048f, -0.3778044879436493f, -0.3667309880256653f, 0.08071848005056381f, -0.865123450756073f, 0.6451484560966492f, -0.5293470621109009f, -1.4819915294647217f, 1.995280146598816f, -0.6921202540397644f, 1.3029942512512207f, -0.1476615071296692f, 0.15694653987884521f, -0.6405912041664124f, -0.8447388410568237f, 0.3645228147506714f, -0.7561046481132507f, 1.417391061782837f, 0.5760944485664368f, 0.13722558319568634f, -0.41890090703964233f, -1.0182898044586182f, 0.448284387588501f, 1.421317458152771f, -0.6346070170402527f, -1.0847539901733398f, -0.5952850580215454f, -0.36421963572502136f, -1.5131834745407104f, -1.9063634872436523f, -1.7777600288391113f, -0.5803453326225281f, -0.6087163090705872f, 0.4910713732242584f, -0.54014652967453f, 1.2756917476654053f, 0.9132878184318542f, -1.101527452468872f, -0.8013262152671814f, -1.9939244985580444f, 1.0690128803253174f, 0.7788016200065613f, 0.35621726512908936f, -0.47195300459861755f, 0.9912806153297424f, -0.21498170495033264f, 1.6797508001327515f, -0.7104259729385376f, 0.06523545831441879f, 0.9159647822380066f, 2.1436846256256104f, 0.4262869358062744f, 0.2694464921951294f, 0.9167301654815674f, 0.6317405700683594f, 0.07572164386510849f, 1.160980463027954f, -0.6461572051048279f, 0.03240058198571205f, -0.7329845428466797f, 0.5573175549507141f, 0.1672649085521698f, -0.30756810307502747f, -0.27301856875419617f, 0.2073671668767929f, 0.1735081523656845f, -0.8468038439750671f, 0.1224641352891922f, 0.020374706014990807f, 1.1085376739501953f, 1.3970056772232056f, -0.5763441920280457f, -1.7790907621383667f, -1.4155789613723755f, -0.11962825059890747f, 1.434804081916809f, -0.6779595613479614f, 0.08594394475221634f, 0.17818263173103333f, 0.8717845678329468f, -0.4843977093696594f, 0.1896892935037613f, 1.0048247575759888f, 0.5574866533279419f, 0.20942160487174988f, -2.0887160301208496f, -1.1161184310913086f, -0.5786607265472412f, 0.8086466789245605f, 0.8664202690124512f, 1.088623046875f, 0.01803939789533615f, 0.968483567237854f, -0.3800842761993408f, 0.3059806525707245f, 0.21581949293613434f, -0.4931473731994629f, -0.15840917825698853f, 0.09949837625026703f, 0.30463987588882446f, 0.9820282459259033f, 2.1544744968414307f, 0.14304663240909576f, -1.460897445678711f, -0.1060129776597023f, -0.37611088156700134f, -0.7917583584785461f, -0.08468178659677505f, 0.8363486528396606f, -1.3825470209121704f, 0.6469754576683044f, 0.5219618082046509f, -0.6058419346809387f, -1.969196081161499f, -0.9246644377708435f, -0.508442759513855f, -1.044813632965088f, 0.353022038936615f, 0.8511560559272766f, -2.0522587299346924f, -0.02700735814869404f, -0.7895724773406982f, -2.2979352474212646f, 1.6485515832901f, -2.076538562774658f, 0.5667335987091064f, 1.3000932931900024f, 0.34126174449920654f, -0.384304404258728f, -0.2445336878299713f, 0.6715037226676941f, -0.3841925263404846f, -0.8124385476112366f, 0.4127030372619629f, 1.1925286054611206f, -1.131065011024475f, 0.3341250419616699f, -0.5277276635169983f, 0.24021567404270172f, -0.18079474568367004f, 0.9263569712638855f, -2.2063262462615967f, 0.6430982351303101f, 1.010899543762207f, -1.5004045963287354f, -0.6776713728904724f, 0.4190317988395691f, -0.1862286925315857f, 0.49416670203208923f, -0.9246350526809692f, 0.5439687371253967f, 1.2584720849990845f, -0.27466103434562683f, -1.1287552118301392f, 0.3993513286113739f, -1.8577489852905273f, 1.882336139678955f, 0.28583550453186035f, -1.3191593885421753f, -0.04691770300269127f, 1.424126148223877f, 0.47822806239128113f, -0.22128069400787354f, -0.16418692469596863f, 1.2590702772140503f, -1.5019725561141968f, -1.2077754735946655f, -0.5713672637939453f, -0.29712751507759094f, -0.08878864347934723f, -1.2249038219451904f, 0.9701720476150513f, 0.02799459546804428f, 2.49943208694458f, -0.5693896412849426f, 0.2992815375328064f, 0.4264316260814667f, -1.8905819654464722f, 0.5416959524154663f, 0.5037822723388672f, 0.06191536784172058f, 0.7051315307617188f, 1.1914323568344116f, 0.7841975092887878f, -0.5143458247184753f, 0.10756650567054749f, 0.03601418063044548f, -1.586909532546997f, -0.36298084259033203f, 0.05457804724574089f, -1.8995989561080933f, -0.4206298589706421f, 1.725341796875f, 0.2990115284919739f, 0.2708130478858948f, 1.202155351638794f, -1.3849509954452515f, 0.37148424983024597f, 1.2834670543670654f, 0.40953299403190613f, 1.5861903429031372f, 2.2006518840789795f, 1.1788570880889893f, 0.04892049357295036f, 0.45504313707351685f, -1.1838287115097046f, 0.2520817816257477f, -1.0377240180969238f, -1.1767886877059937f, 1.5097410678863525f, 0.6526481509208679f, 0.5476107001304626f, -1.4937485456466675f, -0.714340090751648f, -1.4507311582565308f, 0.3059408664703369f, 1.3935821056365967f, -2.0411956310272217f, -1.7664213180541992f, 1.1327776908874512f, 0.7594925165176392f, 0.8204308748245239f, 0.7248584032058716f, 0.6390634179115295f, -0.5571966767311096f, -0.40140989422798157f, -0.22562873363494873f, -0.5097048878669739f, 0.9061921834945679f, -0.15940190851688385f, -0.4242385923862457f, -0.23000852763652802f, -0.5473349094390869f, 0.3962896168231964f, -1.0022584199905396f, -1.9527696371078491f, -0.9625334739685059f, 0.7991358637809753f, -0.010661646723747253f, 0.023252639919519424f, 0.6258861422538757f, 0.3616181015968323f, -0.2700897455215454f, 1.6419692039489746f, -0.7898518443107605f, -0.32855224609375f, 0.9272356629371643f, 0.23767738044261932f, 2.069274663925171f, -0.6138983368873596f, 1.5534266233444214f, 0.562792956829071f, 0.9539335370063782f, 0.34314191341400146f, 0.32635602355003357f, -1.0357731580734253f, -0.021849583834409714f, 0.4444722533226013f, 1.132546067237854f, -1.3696054220199585f, -0.42242521047592163f, 0.6405972242355347f, 0.026323553174734116f, -1.8021960258483887f, -0.17883431911468506f, 0.6420789957046509f, 0.17955924570560455f, -0.2522568106651306f, -1.4123497009277344f, -0.7108830809593201f, -0.39913156628608704f, 0.6917201280593872f, -1.742424488067627f, 0.37175261974334717f, 1.2827181816101074f, -0.1333233267068863f, 2.0086865425109863f, -0.2341606765985489f, 0.7056452035903931f, 0.3473411798477173f, 0.809699535369873f, -3.0800278186798096f, -1.1150798797607422f, 1.4471967220306396f, 0.812851071357727f, 0.4065439999103546f, -0.545810341835022f, -0.6789452433586121f, -0.7672881484031677f, 0.17289385199546814f, 0.40247759222984314f, 0.6123214364051819f, -0.6715307831764221f, 0.18855510652065277f, 1.0615094900131226f, -0.2978079319000244f, -0.4801141619682312f, -1.3616151809692383f, -0.9356100559234619f, 0.978323757648468f, 1.4345933198928833f, 0.956223726272583f, -0.9492759704589844f, -0.9247362613677979f, 1.4304224252700806f, -1.2141680717468262f, 1.5573612451553345f, -2.772749185562134f, -2.07552170753479f, 0.019730882719159126f, -1.6115124225616455f, 0.2347842901945114f, -0.6375554203987122f, -0.2960043251514435f, -0.41874298453330994f, -1.3194098472595215f, 0.05774686485528946f, 0.30748531222343445f, -0.5826377272605896f, -0.31068262457847595f, 0.39234694838523865f, 1.9441745281219482f, 2.0378758907318115f, 0.4360438287258148f, -0.09107904881238937f, -0.24155908823013306f, 0.45299890637397766f, -0.10979556292295456f, 1.179660439491272f, 1.1283513307571411f, -0.8159404993057251f, -0.07127540558576584f, -0.8951465487480164f, 0.31810882687568665f, 0.6084815859794617f, 0.19665859639644623f, -1.3970006704330444f, 1.1369043588638306f, -0.11328133195638657f, 0.12461294233798981f, -0.5991875529289246f, -1.8511996269226074f, 1.1412599086761475f, -1.323111653327942f, -1.642134666442871f, -0.7886322736740112f, 0.016923261806368828f, 0.26853591203689575f, -0.08590231090784073f, -1.8297702074050903f, 1.8586663007736206f, 0.6306189298629761f, -2.0020906925201416f, -0.9767837524414062f, -1.0065470933914185f, 1.5179888010025024f, -0.25139299035072327f, 0.2684777081012726f, 0.2379687875509262f, -1.7342220544815063f, 0.890319287776947f, -0.038704462349414825f, -1.1842435598373413f, -0.6990516185760498f, -0.6046251654624939f, 1.8508250713348389f, -2.0493991374969482f, -1.7228929996490479f, 0.23755493760108948f, 0.9717959761619568f, -0.3798253536224365f, 0.685245931148529f, -1.1771284341812134f, -0.02216072380542755f, -0.1269814372062683f, -0.26579511165618896f, 0.22640825808048248f, -0.760986864566803f, 0.4831203520298004f, 1.5869791507720947f, -0.6875012516975403f, -1.2886475324630737f, 1.2204482555389404f, -0.5210925936698914f, 2.4901089668273926f, 0.15757296979427338f, -0.6784350275993347f, 1.6848376989364624f, -0.762600839138031f, 0.4223840832710266f, 0.05342012271285057f, -1.407547950744629f, 0.7069153189659119f, 0.5444626212120056f, -0.19080407917499542f, 0.14213266968727112f, 0.20559696853160858f, 1.2673413753509521f, 1.375124216079712f, 0.060717396438121796f, 0.31265851855278015f, -0.1293685883283615f, -1.0525683164596558f, -0.5546149015426636f, 1.0222612619400024f, -0.8595789074897766f, 1.0922929048538208f, -0.7460381984710693f, 0.660965621471405f, 0.454408198595047f, -1.4719254970550537f, 0.376058965921402f, -0.7192203998565674f, -0.9688137173652649f, 0.4969656765460968f, 0.42114102840423584f, -0.3883015811443329f, 0.9459248185157776f, 0.22322431206703186f, -0.3906600773334503f, 0.5856817960739136f, 0.2909909188747406f, -1.3024108409881592f, 1.4306825399398804f, 0.4534458816051483f, 0.9106372594833374f, -0.341061532497406f, -0.7916259765625f, -1.1452873945236206f, -0.4609397351741791f, -0.5050092339515686f, 1.0576668977737427f, 0.8928799033164978f, 0.7486283779144287f, 1.0225255489349365f, 1.0325298309326172f, 0.772061824798584f, 0.6529440879821777f, 1.3521305322647095f, 1.9143685102462769f, -1.3935517072677612f, 0.9990993142127991f, 1.297232985496521f, 2.667762517929077f, 0.888896107673645f, 0.8407386541366577f, 1.3103235960006714f, 1.3254046440124512f, 0.7249560952186584f, 1.3489890098571777f, 0.7757742404937744f, 0.16486017405986786f, -1.9583953619003296f, 2.0979738235473633f, -1.1749324798583984f, -1.5510687828063965f, 0.6880179643630981f, -0.27131080627441406f, -2.00603985786438f, -0.14031292498111725f, 0.24766799807548523f, -0.19394119083881378f, -0.08344011753797531f, 1.1331552267074585f, 0.4335167407989502f, 1.659290075302124f, 0.645967423915863f, -0.5915296077728271f, 1.6242163181304932f, -0.9242795705795288f, 1.5664156675338745f, -1.430296540260315f, 0.24254998564720154f, -0.9536606073379517f, 0.2823708951473236f, 0.9416194558143616f, 0.9761032462120056f, -0.14037710428237915f, 0.21578916907310486f, 0.42492198944091797f, 0.9744104146957397f, -0.6820825934410095f, 0.7403110861778259f, -0.014602077193558216f, -1.9035120010375977f, -0.459785133600235f, -0.22833675146102905f, 0.2052573710680008f, 0.060564182698726654f, -0.6736571192741394f, -0.811122477054596f, -0.011842876672744751f}; - -// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g) -// = gemm_Bpack_dram + (nt*(K/4)+g)*64. -static float gemm_Bpack_dram[64*64] __attribute__((section(".data"))) = -{-0.5196930766105652f, 1.8524175882339478f, 1.8365377187728882f, 2.074131727218628f, -0.7373097538948059f, -0.7686780691146851f, -0.05119974538683891f, 1.5985578298568726f, 0.2122737318277359f, 1.1059595346450806f, 1.311963438987732f, 0.424176424741745f, -0.4922901690006256f, 1.6562608480453491f, 0.4111401438713074f, -0.2428770512342453f, 2.081841230392456f, 1.0677173137664795f, -1.4276772737503052f, -0.33180344104766846f, 1.7054011821746826f, 0.6059234738349915f, 1.1122153997421265f, -0.5635794997215271f, -1.364527702331543f, 0.17917323112487793f, 0.5652397871017456f, 0.3271985352039337f, 0.01357425469905138f, 2.691838502883911f, 1.2729166746139526f, -0.6343013048171997f, -0.10098669677972794f, -0.43878018856048584f, -1.9775099754333496f, -0.03591495007276535f, -0.787344753742218f, -0.11653880029916763f, 1.899356484413147f, 0.45238569378852844f, -0.5850009918212891f, 0.17411360144615173f, -0.2031504362821579f, -1.271364688873291f, 0.8729038834571838f, 0.14136412739753723f, -1.4058030843734741f, 0.4083256125450134f, -1.3799183368682861f, -0.21034009754657745f, 0.8919094204902649f, 0.18319325149059296f, 0.5042280554771423f, -1.007485270500183f, -1.3393090963363647f, -0.4502287805080414f, -0.16011619567871094f, -1.608611822128296f, 0.1806594282388687f, 1.864193320274353f, -1.201043963432312f, -0.07190026342868805f, 0.14386875927448273f, -0.1494988203048706f, 0.6289454698562622f, -0.0685209259390831f, -1.2115036249160767f, -0.5285655856132507f, -1.5568547248840332f, -1.2001843452453613f, 0.2736855447292328f, -0.6129935383796692f, 0.9846591353416443f, 1.6213051080703735f, 0.5066256523132324f, -0.126494362950325f, 0.03450252488255501f, 1.3147404193878174f, -0.6877241134643555f, -0.10271778702735901f, -1.401864767074585f, -0.134507417678833f, -0.5799044966697693f, 1.0302584171295166f, 0.16683164238929749f, 0.13005995750427246f, 0.3381371796131134f, 1.025931715965271f, -1.1549469232559204f, 0.21058911085128784f, 0.0752936601638794f, 0.4396262466907501f, 0.029413584619760513f, 0.18214739859104156f, 0.03471093624830246f, -1.8955014944076538f, 0.9624823927879333f, 0.04533285275101662f, -2.03769850730896f, -1.0922635793685913f, -0.17383840680122375f, -1.065432071685791f, 0.926749587059021f, -0.16234397888183594f, 0.36853712797164917f, -0.2603956162929535f, 0.12695534527301788f, -1.7026076316833496f, 0.6422180533409119f, -0.6823133230209351f, -1.6458345651626587f, 0.15697401762008667f, -1.2265124320983887f, 0.7861263155937195f, -2.095715284347534f, 0.4538240432739258f, -0.4390734136104584f, 0.4836112856864929f, 0.9263717532157898f, 0.06930986046791077f, 1.4818024635314941f, -0.9132123589515686f, -1.606928825378418f, -0.5966755747795105f, 0.7912061810493469f, 1.1138908863067627f, 1.4326399564743042f, 0.34999290108680725f, 0.7664128541946411f, 0.4324735403060913f, -1.018404245376587f, 0.012393372133374214f, 0.6695152521133423f, 1.2033629417419434f, 0.8298293352127075f, -0.9191604256629944f, 0.958231508731842f, 0.2724153697490692f, 0.9915789365768433f, 0.3825709819793701f, -0.5029784440994263f, 1.5228251218795776f, -2.502131938934326f, -0.6252055764198303f, -0.5682758688926697f, -1.4246463775634766f, 0.4983586370944977f, -1.3945032358169556f, -2.804393768310547f, 0.4811320900917053f, 0.3296896517276764f, 1.4936070442199707f, 1.691656231880188f, 0.1595890074968338f, 1.024785041809082f, 0.12853768467903137f, -0.46378275752067566f, 1.3842717409133911f, -0.6843701601028442f, -0.3655944764614105f, -0.34340038895606995f, 0.9877837896347046f, 1.3732529878616333f, -0.32952919602394104f, -0.540728747844696f, 0.5175154209136963f, -1.1346107721328735f, 0.4711856544017792f, 1.920310378074646f, 0.4751400351524353f, 0.5673654675483704f, 0.2462792843580246f, 2.888267755508423f, -0.5556330680847168f, 0.017246929928660393f, -0.3958292305469513f, -0.3625917434692383f, 1.179726004600525f, 0.8268131613731384f, 1.457729697227478f, -1.029435634613037f, -1.2957319021224976f, 0.2842133045196533f, -2.20194673538208f, 0.5491127967834473f, -1.2930694818496704f, 0.4510805904865265f, -0.3953653573989868f, 1.7161552906036377f, 0.15392881631851196f, -1.4651801586151123f, -0.5170696377754211f, -0.789655864238739f, -1.313944935798645f, 1.211775779724121f, 0.9332025647163391f, -0.2605237364768982f, 1.5151708126068115f, -1.521173357963562f, -1.1669979095458984f, -2.0334060192108154f, -0.08260460197925568f, 1.4479997158050537f, 0.1535591334104538f, -0.19014781713485718f, -0.9898110032081604f, 0.3790753483772278f, 1.921068549156189f, -0.13450416922569275f, 0.5259649753570557f, 1.2340812683105469f, 0.19075801968574524f, -0.19310960173606873f, 1.5767059326171875f, 0.11904352903366089f, 0.2899794280529022f, -2.0928122997283936f, -0.0447654202580452f, 0.11111800372600555f, 0.38851943612098694f, 0.5478768348693848f, -1.3495665788650513f, -0.44743525981903076f, -0.7099960446357727f, 0.9817765355110168f, 1.0983757972717285f, -0.25671127438545227f, -2.2621774673461914f, 0.16864269971847534f, -1.6143133640289307f, -0.011093960143625736f, 1.2098065614700317f, -1.4476906061172485f, 1.3612878322601318f, -1.3062708377838135f, 1.6407432556152344f, -0.33931660652160645f, 0.13347899913787842f, -0.9141297936439514f, -0.15479597449302673f, -0.699771523475647f, -1.096972107887268f, -0.55166095495224f, -0.3263216018676758f, 0.6205294132232666f, -1.4098610877990723f, -0.41539478302001953f, 3.068681478500366f, 0.3701395094394684f, -0.7284356355667114f, -0.9289583563804626f, 0.8983376026153564f, -0.48385927081108093f, 0.032833296805620193f, -0.10948953032493591f, 0.46667027473449707f, 0.6613953113555908f, -0.03297487273812294f, -0.13994985818862915f, 1.0257868766784668f, -1.9717178344726562f, 0.5998077988624573f, 1.361150860786438f, -0.173064723610878f, 0.7555463910102844f, -0.14636565744876862f, -0.09429822862148285f, 0.29657915234565735f, 0.11924134939908981f, 0.1167483851313591f, -0.5673035979270935f, -1.7532938718795776f, -1.0694472789764404f, 0.08452916890382767f, 0.33887022733688354f, -1.223060965538025f, 0.19821691513061523f, 0.38515371084213257f, -0.07315804064273834f, 0.8780375719070435f, 0.12963341176509857f, -0.6663824319839478f, 0.4921484887599945f, -0.9834228754043579f, -1.2953417301177979f, -1.5002580881118774f, 1.1558160781860352f, -0.7528656125068665f, -1.160732388496399f, -1.486362099647522f, 0.13496126234531403f, 1.354621410369873f, 1.9504122734069824f, 1.0814120769500732f, 0.7843427658081055f, 1.8615505695343018f, 0.6727956533432007f, -0.4322131276130676f, 1.8005059957504272f, 1.3338027000427246f, -0.3184575140476227f, 1.1182889938354492f, 1.8907326459884644f, 0.19988827407360077f, 0.9396787285804749f, 1.0826958417892456f, -0.6519155502319336f, 0.2863307297229767f, 0.10145679116249084f, 0.09791278839111328f, -0.8082833886146545f, 1.441809892654419f, -0.6460452675819397f, 0.04811352491378784f, 1.382570505142212f, -0.6665758490562439f, 0.7876792550086975f, -0.15957334637641907f, -0.10303135216236115f, 0.27407389879226685f, -0.5971476435661316f, -1.0348221063613892f, 0.42101818323135376f, -0.548947811126709f, -1.2540193796157837f, -1.2696188688278198f, 0.6492337584495544f, 0.2340298593044281f, -0.1553155928850174f, -0.4344737231731415f, -1.0962450504302979f, -1.122981309890747f, -1.865479826927185f, -2.1207547187805176f, -0.6719433665275574f, -0.7559810280799866f, 0.1943359375f, 1.0423648357391357f, -1.1257092952728271f, -0.04083564877510071f, 0.7159355282783508f, 0.3418067395687103f, -0.6778577566146851f, 0.17783714830875397f, -0.16209769248962402f, -0.026765741407871246f, -0.33800795674324036f, 0.6160297393798828f, -1.5755101442337036f, -1.1856271028518677f, 0.3281179666519165f, -0.03522142022848129f, -0.5061670541763306f, 0.8149546980857849f, 0.6836134195327759f, -1.2236212491989136f, 0.9091914892196655f, -0.6015433669090271f, 0.8525826930999756f, -0.6866192817687988f, 1.1305443048477173f, -0.3458130359649658f, -0.673958420753479f, -0.16243989765644073f, 1.8315014839172363f, -0.7265217900276184f, -0.44100794196128845f, -1.1563644409179688f, -1.1872515678405762f, -0.83109050989151f, -1.1144185066223145f, 1.2617039680480957f, 0.6814237833023071f, 2.0168917179107666f, 0.481290727853775f, 0.313506156206131f, -0.5229234099388123f, -0.26092126965522766f, -2.14499831199646f, 1.0247118473052979f, 2.1447153091430664f, -0.34384623169898987f, 0.9131002426147461f, -2.4607415199279785f, 1.2518025636672974f, -0.35988521575927734f, -0.43994462490081787f, 1.9879144430160522f, -0.692777693271637f, 0.4603208601474762f, 0.6009213924407959f, 0.8063201308250427f, 0.979621946811676f, 0.9170622229576111f, -0.11666277050971985f, 0.2106197327375412f, 0.5228466987609863f, -1.7686034440994263f, -0.8117372393608093f, -0.877277135848999f, 0.9701831936836243f, -1.7244060039520264f, 1.61734139919281f, -0.9298601746559143f, 0.41942062973976135f, 2.0680532455444336f, -1.2342560291290283f, 2.678502321243286f, 0.3004297614097595f, -1.303833246231079f, 0.7512189745903015f, 0.3729371726512909f, 0.17139992117881775f, 0.5995815396308899f, -0.19726276397705078f, -0.11891679465770721f, -1.0388710498809814f, -0.1431705802679062f, 1.1298640966415405f, 0.587632417678833f, 0.020525068044662476f, 0.12478796392679214f, 0.2550857663154602f, -0.4017896056175232f, -1.7806396484375f, -0.08095061779022217f, -1.759868860244751f, 0.820906937122345f, -0.15008316934108734f, -1.8686583042144775f, -0.6503500938415527f, 0.6934462189674377f, 1.4084579944610596f, -2.2690858840942383f, 0.39256006479263306f, -0.4266415536403656f, 0.36491602659225464f, 0.807181715965271f, -1.5602378845214844f, -0.055706802755594254f, -0.712527871131897f, 0.2629498243331909f, 1.1325268745422363f, 0.8828094601631165f, 0.9763681888580322f, 0.35255447030067444f, -0.5103265047073364f, 0.048219867050647736f, -0.8320447206497192f, 0.9133552312850952f, 0.35530006885528564f, 1.5981814861297607f, 0.6848441362380981f, -0.238090381026268f, -0.8003812432289124f, 0.22974559664726257f, -0.19079573452472687f, -0.364820271730423f, -0.658343493938446f, -1.2190016508102417f, -1.0552600622177124f, -0.9370644688606262f, 0.07260560989379883f, 0.24500113725662231f, 1.690338134765625f, 1.8711999654769897f, 1.2940467596054077f, 0.49941954016685486f, -0.5331865549087524f, -0.26645612716674805f, 0.3932536244392395f, 0.45422324538230896f, 0.4487703740596771f, -0.43473342061042786f, 1.8583357334136963f, 0.35827821493148804f, 0.1845020353794098f, 1.472187876701355f, 0.9056950807571411f, -2.0051839351654053f, -1.232080101966858f, 1.510047435760498f, -0.4955300986766815f, -1.451595664024353f, 1.1304244995117188f, -0.23605898022651672f, -1.4099090099334717f, 0.033967722207307816f, -0.27385643124580383f, 0.052069131284952164f, 0.6194980144500732f, 1.2993793487548828f, 0.37411054968833923f, 0.29060807824134827f, -0.14929574728012085f, 0.16823826730251312f, 1.7945916652679443f, -0.8427640199661255f, -0.776123583316803f, 0.7065202593803406f, 0.020995743572711945f, -0.7148111462593079f, -0.21817894279956818f, 0.6545652151107788f, -1.3288897275924683f, -0.5552407503128052f, 0.869994044303894f, 0.17624177038669586f, 0.10479936003684998f, 1.1782784461975098f, -0.8539279699325562f, 0.18701297044754028f, -0.3077951669692993f, 1.3164128065109253f, 0.7719241976737976f, 2.0046799182891846f, -0.05154263228178024f, -0.09019286930561066f, 0.9585400819778442f, -1.2275644540786743f, 0.35432279109954834f, -0.2604968845844269f, 0.6541568636894226f, -0.43890517950057983f, -0.05930287018418312f, -1.202606201171875f, -1.2769453525543213f, -0.3968345820903778f, 0.5857738256454468f, -0.3441990613937378f, 0.6814432144165039f, 0.772460401058197f, 0.007647526450455189f, -0.25153985619544983f, 0.84389328956604f, -0.2621386647224426f, -0.42434096336364746f, -0.6080525517463684f, 0.011438353918492794f, 0.0011970907216891646f, 0.672935426235199f, -0.6712173819541931f, -0.2299986481666565f, 0.13960936665534973f, 0.6251678466796875f, -1.8366930484771729f, -0.45854899287223816f, 0.49610286951065063f, -0.8524967432022095f, -0.19241304695606232f, 0.47062209248542786f, -0.7075293660163879f, 0.01930052787065506f, 0.7421151995658875f, -2.1299924850463867f, -1.8519654273986816f, -0.789726972579956f, -1.3833003044128418f, -1.1265941858291626f, 0.4917159378528595f, 0.744454026222229f, -0.761397659778595f, 1.7994297742843628f, 0.6710255146026611f, 1.5392361879348755f, -0.8462389707565308f, -1.374268651008606f, -0.29935309290885925f, -1.3537677526474f, -0.9374998211860657f, 0.966241180896759f, 2.2905983924865723f, 0.4570314586162567f, 1.168905258178711f, -0.22257985174655914f, 1.2293546199798584f, -0.4278397262096405f, -0.527760922908783f, -1.798736572265625f, -0.07677789032459259f, 0.781619668006897f, 0.13910506665706635f, -1.7339553833007812f, 0.7391383647918701f, 1.4115384817123413f, 0.38143935799598694f, -0.051372598856687546f, 0.867921769618988f, -1.1712745428085327f, 1.219971776008606f, -0.9404910206794739f, 0.6608859300613403f, 0.49976596236228943f, 0.9063552021980286f, -0.273310124874115f, -0.8444989919662476f, 0.06740748137235641f, -1.1771124601364136f, -2.319305896759033f, 1.0881538391113281f, 1.7532049417495728f, -1.352665662765503f, -1.717094898223877f, 0.539482057094574f, -0.4647720456123352f, 0.15511354804039001f, -0.933184802532196f, 2.0511553287506104f, 1.7320348024368286f, 0.39241987466812134f, -0.3704788386821747f, 0.8811249136924744f, 0.537391722202301f, -0.48424550890922546f, -1.1632671356201172f, 0.3275741636753082f, -1.6853573322296143f, -1.058225393295288f, 1.4543863534927368f, -0.2975543737411499f, -0.8741039633750916f, -1.0332050323486328f, -0.41522952914237976f, 1.5585501194000244f, 2.5905730724334717f, -0.14419972896575928f, -0.47694188356399536f, -0.2289033830165863f, 0.3704476058483124f, 1.024267315864563f, 0.35092905163764954f, -0.11602573096752167f, -0.5128945112228394f, 0.8613727688789368f, 2.617750883102417f, 0.8522545695304871f, 0.3442921042442322f, -0.21941426396369934f, 1.6952100992202759f, 0.7312605381011963f, -2.2600386142730713f, -1.3617544174194336f, 1.5765711069107056f, 0.1465187966823578f, 1.8045871257781982f, 1.3033205270767212f, 0.18405571579933167f, -1.9564915895462036f, 0.1487964242696762f, -0.9366428256034851f, -0.950509250164032f, -0.2993629574775696f, -0.6517760157585144f, 0.2768969237804413f, -0.006574048660695553f, -1.8791542053222656f, -1.5537861585617065f, -0.9000288248062134f, 1.180068850517273f, -1.0849788188934326f, 0.6662015318870544f, -0.7253139615058899f, 1.3528261184692383f, -0.1364641636610031f, 1.1062331199645996f, -0.2775273025035858f, -0.7301269769668579f, -0.9058868288993835f, 0.3395512104034424f, 0.954081654548645f, -0.042208630591630936f, 0.05911993607878685f, -0.3268890678882599f, 0.8507034778594971f, -0.18483993411064148f, -1.1319291591644287f, -0.03408394008874893f, 1.5095572471618652f, -0.11943084001541138f, 1.4642930030822754f, 0.8833335041999817f, 1.1377960443496704f, 0.7793198823928833f, 0.07684005051851273f, -1.4407286643981934f, 0.5401078462600708f, -0.13687263429164886f, -0.4707719087600708f, -0.23642085492610931f, 2.775019645690918f, -0.048772864043712616f, 0.9577808976173401f, -0.12049184739589691f, -0.008940046653151512f, -1.624023199081421f, -0.5534864068031311f, 0.8154556751251221f, -0.03924920782446861f, 1.350463628768921f, -0.04840381070971489f, -0.7114460468292236f, -0.7299661040306091f, -1.4113084077835083f, 0.34884995222091675f, -0.43344026803970337f, 2.270435094833374f, 2.0011656284332275f, -2.1169581413269043f, 0.3513871729373932f, -0.9149423837661743f, -0.28648310899734497f, 1.87372624874115f, 0.07205427438020706f, 0.0007797196158207953f, -0.14003179967403412f, 0.9105306267738342f, -1.0649497509002686f, 0.45754849910736084f, 0.15436618030071259f, -0.744165301322937f, -0.51121985912323f, 0.23932333290576935f, 0.9544809460639954f, 0.16341786086559296f, -0.30999821424484253f, 0.15972714126110077f, 0.8302736878395081f, -1.421786904335022f, -0.09449474513530731f, 0.07646692544221878f, 0.15851546823978424f, -0.38410186767578125f, -1.4129188060760498f, -1.0182603597640991f, 0.4253365993499756f, 0.98757004737854f, 1.1435766220092773f, 0.042469993233680725f, -0.12866179645061493f, -0.3201303780078888f, 1.3020575046539307f, 0.05820586159825325f, -1.411534309387207f, -0.5787767171859741f, -1.1051533222198486f, 1.3300304412841797f, -0.41862916946411133f, -0.43831324577331543f, 1.0452755689620972f, 0.7565826773643494f, 1.3442822694778442f, -0.005769374314695597f, 1.3893702030181885f, -1.459441065788269f, 0.7733556032180786f, 1.281721591949463f, 1.922197699546814f, 1.5146862268447876f, -0.9400398135185242f, 1.7596426010131836f, 0.4703059196472168f, -1.1514455080032349f, -0.6767635941505432f, 0.9793415665626526f, 0.7851172089576721f, 0.9229853749275208f, -1.3558963537216187f, -0.4045674800872803f, -0.022665904834866524f, 0.04516039043664932f, -0.0492362417280674f, 0.688543975353241f, -0.6404033303260803f, -0.8518562912940979f, 0.5739071369171143f, -1.0256696939468384f, 0.5530444383621216f, 1.2585986852645874f, 0.23174597322940826f, -1.7316545248031616f, -0.300735741853714f, 0.8933703303337097f, -0.2885432839393616f, -0.7255284190177917f, 0.17296810448169708f, -1.597967267036438f, -0.8826010227203369f, -0.7285105586051941f, 1.0816147327423096f, 0.689936101436615f, 0.1465923935174942f, -0.2598642110824585f, 1.2287657260894775f, 0.4021441638469696f, 0.29235512018203735f, -1.4843645095825195f, -1.288714051246643f, 0.8229716420173645f, -1.8816033601760864f, -0.24393519759178162f, 1.1026350259780884f, 0.12240499258041382f, -0.6282588839530945f, -0.6169579029083252f, 0.8369725942611694f, -0.21276021003723145f, -1.037753939628601f, -0.5319015383720398f, -1.4479600191116333f, -0.7402247190475464f, -1.9904698133468628f, -0.8090559244155884f, -2.649178981781006f, 0.6452047228813171f, 1.6631805896759033f, -0.6097846031188965f, -0.5651611685752869f, -0.09456146508455276f, 0.2765144407749176f, 0.2960791289806366f, 1.2084519863128662f, -2.0801830291748047f, 0.8232543468475342f, -0.5484461784362793f, 0.34564146399497986f, 1.220015525817871f, -0.43524736166000366f, -1.7363234758377075f, -1.441352128982544f, 1.4536579847335815f, 1.2042756080627441f, -0.9513401389122009f, 1.7534879446029663f, -1.5643774271011353f, 0.2642883360385895f, 0.5893656611442566f, -0.6958070397377014f, -0.9822604656219482f, 0.9073857069015503f, 0.13120052218437195f, 0.3665435016155243f, 0.8943644165992737f, 0.990159809589386f, 1.0685539245605469f, 0.88179612159729f, 0.11932405829429626f, -1.6899914741516113f, -0.6570671200752258f, 1.6827791929244995f, 0.4652342200279236f, -0.9479421973228455f, -0.527360200881958f, -0.17596317827701569f, 1.684232473373413f, -0.005598508287221193f, -1.2838374376296997f, 0.16975225508213043f, 0.8452434539794922f, 0.28028908371925354f, 0.920665442943573f, 1.9825818538665771f, -1.0555790662765503f, -0.7873882055282593f, -0.31694626808166504f, -0.1027197390794754f, -1.1638461351394653f, -0.17423255741596222f, -0.9778326153755188f, 0.07384445518255234f, 0.2161293476819992f, -2.4143142700195312f, 0.6280608177185059f, 0.604034423828125f, -0.06171112880110741f, -0.27243953943252563f, 2.0528199672698975f, -0.6635909676551819f, 0.4586471915245056f, -1.6872135400772095f, 0.03186811879277229f, 2.094301462173462f, -0.38392743468284607f, 0.5313023328781128f, 2.5650343894958496f, -0.06663789600133896f, -0.08388976007699966f, 0.5069190263748169f, -0.9045271277427673f, -0.13790902495384216f, 1.3361674547195435f, 1.065782904624939f, 0.9382249712944031f, -0.5762946009635925f, 0.26595842838287354f, -0.515076756477356f, 1.8256999254226685f, -0.057207878679037094f, -0.2434438019990921f, 0.9149488806724548f, -2.118633508682251f, -1.2563103437423706f, 1.8411842584609985f, 0.8629192113876343f, -1.979547142982483f, 0.4109382629394531f, 0.054416440427303314f, 0.24019105732440948f, -0.3381706476211548f, -1.1024436950683594f, 0.2338137924671173f, -0.4720325171947479f, -1.3284554481506348f, 0.5184157490730286f, -0.48354241251945496f, 1.3702332973480225f, 0.18823130428791046f, -1.2953920364379883f, -0.13154013454914093f, 0.08959710597991943f, -0.44053706526756287f, 0.20576980710029602f, -1.0061860084533691f, 1.1124646663665771f, -0.7451666593551636f, -0.9709926247596741f, -0.005489187315106392f, 0.8581770658493042f, 0.8032210469245911f, -0.15868736803531647f, 1.6983613967895508f, -0.055956143885850906f, -0.45469725131988525f, 1.2942347526550293f, 0.1953326016664505f, -1.775277853012085f, 1.508985161781311f, -0.47618457674980164f, 1.1132714748382568f, 2.190094470977783f, 0.3530663549900055f, 0.4473947584629059f, 0.7191661596298218f, -1.9299876689910889f, 0.9975112676620483f, 1.1845413446426392f, 1.3882135152816772f, 0.1371254324913025f, -0.84199458360672f, -1.6440988779067993f, -0.06394191086292267f, -0.201401948928833f, 0.2995571792125702f, -1.3531256914138794f, 0.8332006335258484f, -0.1304839849472046f, 0.5836395621299744f, -0.38578328490257263f, -1.6579194068908691f, -0.2272949367761612f, 0.6312239170074463f, -1.346675992012024f, 0.6521760821342468f, -1.3508291244506836f, 2.1090853214263916f, -0.8325138688087463f, -0.9411794543266296f, -0.8664024472236633f, 0.3974232077598572f, 0.33468785881996155f, 0.23106975853443146f, -1.2114311456680298f, -1.2591102123260498f, -0.010216501541435719f, 0.9779312610626221f, 0.0748591348528862f, -1.8819596767425537f, 0.27785420417785645f, 0.734195351600647f, -0.37356483936309814f, -0.395193487405777f, -1.2448596954345703f, -0.4259602427482605f, -0.9261072874069214f, 0.33485373854637146f, 1.0248674154281616f, -0.6913897395133972f, -0.5325083136558533f, -2.0061066150665283f, 0.5374981760978699f, 0.4246957302093506f, -0.39820799231529236f, 0.4305093586444855f, 0.7205038070678711f, -0.2832542359828949f, 0.09373100101947784f, 1.657375454902649f, 1.129982829093933f, 0.6332632303237915f, 0.23538991808891296f, 0.08460769802331924f, 1.6273547410964966f, 1.4943406581878662f, 0.4516901969909668f, 0.681415319442749f, -2.738783597946167f, -0.04374435544013977f, -0.13210587203502655f, -0.20210233330726624f, -1.3241872787475586f, -0.1972821205854416f, 0.29198724031448364f, -1.2132630348205566f, 0.7370628118515015f, 0.9276220798492432f, 0.6291629076004028f, -0.23619088530540466f, -0.02293848805129528f, -2.780918836593628f, -0.8419364094734192f, -0.6721378564834595f, 0.13255943357944489f, -1.203613042831421f, 1.333530068397522f, -2.03551983833313f, 0.8634518384933472f, -1.4491990804672241f, 0.3142701983451843f, -1.005286455154419f, -1.3435431718826294f, 1.2677130699157715f, -1.2937111854553223f, -0.7414584755897522f, -0.3299030065536499f, 0.3301123380661011f, 0.9814369082450867f, -1.4912174940109253f, 0.5385298132896423f, 1.3361884355545044f, -0.5637743473052979f, 0.663472592830658f, 0.5684458613395691f, 0.5110347270965576f, 0.9753285646438599f, 1.9773973226547241f, -1.2740811109542847f, -0.7873809337615967f, 2.1380560398101807f, 0.3426303565502167f, 0.9504527449607849f, -1.5345426797866821f, -0.17094235122203827f, 0.9801839590072632f, -0.05862395092844963f, 2.073120594024658f, -0.3976811170578003f, -0.20158079266548157f, 1.4582887887954712f, -0.5315611362457275f, 0.8970864415168762f, -1.1392240524291992f, -1.309956669807434f, -0.0961947962641716f, -2.4108095169067383f, -0.009612545371055603f, -0.6612817049026489f, 0.8028563857078552f, -0.0704915001988411f, -1.5173195600509644f, 0.20034420490264893f, -0.24687924981117249f, 0.23627454042434692f, 0.7618011236190796f, -0.4506336748600006f, 0.3065869212150574f, 0.9810793399810791f, -0.5398741960525513f, -0.42057791352272034f, 0.4069875180721283f, -0.3191024363040924f, 0.8826714158058167f, 0.6646241545677185f, -2.1909663677215576f, 0.4844568967819214f, 0.6883501410484314f, 1.6444298028945923f, -0.09725860506296158f, -0.972576379776001f, 0.9053502678871155f, 0.3767105042934418f, 0.6639789342880249f, 0.05910481512546539f, 0.6989551782608032f, -0.16035781800746918f, -1.092200756072998f, 0.02478332258760929f, 0.4938628077507019f, -0.5733305811882019f, 0.6898083090782166f, 0.6767069697380066f, 0.5416850447654724f, -0.17832504212856293f, -0.7599223852157593f, 0.4881221354007721f, -1.4561625719070435f, 0.5495442748069763f, 0.1979023516178131f, -0.33105704188346863f, -1.3217003345489502f, -0.24350062012672424f, -0.4956028163433075f, 1.6506794691085815f, 0.7990935444831848f, 0.6958364844322205f, -0.9322998523712158f, 0.18523266911506653f, 0.039212681353092194f, -1.0234346389770508f, 0.31960463523864746f, -0.8433935046195984f, -2.1094107627868652f, -0.36244451999664307f, -0.12838764488697052f, -0.30779823660850525f, -1.713757038116455f, 0.23439815640449524f, -0.40255531668663025f, -1.402301549911499f, -0.8660640120506287f, 0.8337363600730896f, 0.1786288470029831f, 1.002273440361023f, -0.676571249961853f, 0.02751591056585312f, -1.0370279550552368f, 1.4250038862228394f, 0.3661484718322754f, 0.2926573157310486f, 0.9981994032859802f, 0.9345365762710571f, 0.08568228036165237f, -0.5905015468597412f, -0.603374183177948f, 0.6896741390228271f, 0.7225849628448486f, -0.4239010512828827f, 1.3261003494262695f, -1.067254662513733f, 0.6243192553520203f, -2.4506609439849854f, 1.3008989095687866f, -1.3570464849472046f, 0.4011816382408142f, -0.4519657790660858f, -0.5720081329345703f, 1.7311065196990967f, 0.40453165769577026f, 0.5910705924034119f, 0.5206778049468994f, 0.26302585005760193f, 1.462548851966858f, 0.806793212890625f, 2.1060609817504883f, 0.1250244528055191f, -1.0741941928863525f, -0.04460187256336212f, -0.7740861177444458f, 1.4476147890090942f, -1.8094489574432373f, 0.40884074568748474f, -0.13771961629390717f, -0.48222219944000244f, -0.9596375823020935f, 0.88399738073349f, -1.1032230854034424f, 0.4613407552242279f, -0.6266279220581055f, -0.059072766453027725f, 1.432356357574463f, -0.3214779198169708f, -0.32065340876579285f, -1.381635308265686f, -0.36659157276153564f, -1.3767681121826172f, -0.6575090289115906f, -1.0047813653945923f, 0.15763555467128754f, -2.729581594467163f, -0.12834908068180084f, -0.4433963894844055f, -1.7998695373535156f, -0.4873202443122864f, -0.360196590423584f, -0.2588382959365845f, 1.3558588027954102f, -0.6274254322052002f, 0.47283434867858887f, -0.6854686141014099f, 1.4168455600738525f, 0.8578910827636719f, 0.8829494118690491f, -0.8793548941612244f, -0.8100994825363159f, 0.6788665652275085f, 0.33716168999671936f, -0.5290454030036926f, 0.9324967265129089f, 0.6762214303016663f, 0.14378660917282104f, 2.18232798576355f, 0.037647929042577744f, 0.21775013208389282f, 2.395190954208374f, 0.4023849368095398f, 1.6874338388442993f, -2.743908405303955f, -1.9040486812591553f, 0.03124547004699707f, 0.3133164942264557f, -1.2002359628677368f, -0.22739538550376892f, -0.23840101063251495f, -0.682720422744751f, 0.46212872862815857f, -0.9165003299713135f, -1.665152907371521f, 0.7060098052024841f, 0.9033956527709961f, -0.3824882507324219f, 0.1978939324617386f, -0.032255567610263824f, -0.9255485534667969f, 1.1699923276901245f, -1.751257061958313f, 0.9016333222389221f, -1.3840047121047974f, -0.8077333569526672f, 1.2065165042877197f, 0.1894015073776245f, -0.6532776355743408f, -0.2359209805727005f, 0.6057451367378235f, -0.3558703064918518f, 1.4644410610198975f, -0.8236719965934753f, -0.3576895296573639f, -0.9380688667297363f, 1.4365062713623047f, 1.5855119228363037f, 0.39503413438796997f, -0.09881415218114853f, -1.8138774633407593f, -0.6933608055114746f, 1.3770830631256104f, 1.0369852781295776f, 0.522045910358429f, -2.008735179901123f, -0.7871567606925964f, 0.07242458313703537f, -1.9583851099014282f, -0.1682676076889038f, -0.09407275170087814f, -1.5021157264709473f, 0.9083237051963806f, -0.627318799495697f, -1.1001535654067993f, 0.8703535795211792f, 0.9143402576446533f, -0.9333758354187012f, -1.9243427515029907f, 0.5441842079162598f, 1.674156904220581f, -0.1457550972700119f, -0.663499116897583f, 1.2121490240097046f, 0.9221742749214172f, 0.5710628032684326f, -0.7049512267112732f, 2.053586959838867f, 0.9570578932762146f, -1.0222344398498535f, -0.5004768371582031f, -0.8111429214477539f, 2.0269458293914795f, -2.568801164627075f, 1.0234973430633545f, -0.5307855606079102f, 2.509202480316162f, 0.9496951103210449f, 1.4498697519302368f, 1.9099774360656738f, -0.41487377882003784f, -2.154376745223999f, -1.1843920946121216f, -0.13271500170230865f, -0.6129738092422485f, 1.0654757022857666f, 0.589937686920166f, -1.330565333366394f, 1.276455044746399f, -1.1509960889816284f, 1.4542961120605469f, 0.21978320181369781f, -0.26515746116638184f, 0.7749262452125549f, 0.12985540926456451f, 1.0439496040344238f, -1.7788974046707153f, 0.3559875190258026f, -0.05403977632522583f, 0.07127834856510162f, -0.8058504462242126f, -0.35074129700660706f, -0.17826782166957855f, -0.2625838816165924f, 0.7513576745986938f, 0.18278615176677704f, -0.9977543950080872f, 0.45807182788848877f, 0.7924372553825378f, -0.9643475413322449f, 0.46591266989707947f, 1.3539252281188965f, -1.459295630455017f, 0.37469375133514404f, 0.7460726499557495f, 0.26590028405189514f, -1.0854178667068481f, -1.5084643363952637f, 1.1576530933380127f, -0.21968594193458557f, 0.4215473532676697f, -0.32665783166885376f, 2.3412764072418213f, -0.5351880192756653f, 1.0548261404037476f, -0.4031357765197754f, 0.6454427242279053f, -0.6314883828163147f, -1.2809436321258545f, 0.009232764132320881f, 1.2330046892166138f, 0.6238497495651245f, 1.5213422775268555f, -0.4976206421852112f, -1.0449823141098022f, -0.27992430329322815f, 1.2101659774780273f, -0.9885204434394836f, 0.350207656621933f, -0.2644238770008087f, -1.0631657838821411f, -0.7474985122680664f, 0.3266002833843231f, -0.49929699301719666f, -1.4715039730072021f, 0.5023646950721741f, 0.4279913902282715f, 1.0040385723114014f, 1.708235263824463f, -0.2667044997215271f, 0.18906556069850922f, 1.0375696420669556f, 0.7793638706207275f, -0.6498270630836487f, -0.48095330595970154f, 0.13058727979660034f, 0.8354679346084595f, 1.6123450994491577f, -0.4939593970775604f, 0.30456098914146423f, 2.1306238174438477f, 0.026654772460460663f, 0.7856671810150146f, 0.8128212690353394f, 0.04477962106466293f, -1.7439521551132202f, -1.1074808835983276f, -0.6043308973312378f, 0.0144581263884902f, -0.2186170518398285f, -1.0052590370178223f, 1.2696419954299927f, 1.212203025817871f, -0.3618496060371399f, 1.1163915395736694f, 0.4140532910823822f, 0.12140734493732452f, 0.836719274520874f, -0.6638585925102234f, -2.0399112701416016f, -0.6206883192062378f, -0.9584940075874329f, -0.45995032787323f, 0.06401844322681427f, -0.4596543312072754f, -0.5983195900917053f, 0.844002366065979f, -0.6386957168579102f, -1.218558430671692f, -0.1842191219329834f, -1.7404849529266357f, -0.43988513946533203f, 0.978778600692749f, 1.003504991531372f, 1.839215874671936f, -0.7032709121704102f, -2.1381378173828125f, 0.3657694160938263f, -0.01325446367263794f, 1.1153861284255981f, -0.4465695023536682f, 1.315346598625183f, 1.038090705871582f, 2.2033896446228027f, -0.907842755317688f, 2.0937135219573975f, -0.9239965677261353f, -0.553339421749115f, 0.09259668737649918f, 1.0563279390335083f, 0.8357012271881104f, 0.39962950348854065f, -0.5224289298057556f, 1.8076858520507812f, 0.973744809627533f, 1.072774887084961f, -1.0264922380447388f, -1.1302908658981323f, -0.2908953130245209f, -1.6867245435714722f, 1.5600812435150146f, 1.8405765295028687f, -0.7730143070220947f, -0.2413354218006134f, 2.3902435302734375f, -0.5483986139297485f, 0.643738329410553f, 0.7495583891868591f, -1.2053273916244507f, 1.4264872074127197f, -1.1080999374389648f, -0.1472785621881485f, -2.2464029788970947f, 0.40276598930358887f, 0.1035822331905365f, 0.3125080168247223f, 0.19359846413135529f, -0.40304437279701233f, 0.7284680604934692f, 1.4359182119369507f, -1.437017560005188f, 0.5239248275756836f, -0.26981568336486816f, 0.017309220507740974f, 1.2778736352920532f, -0.30529654026031494f, 0.7788206934928894f, -0.08426624536514282f, 0.8460555672645569f, 1.1856203079223633f, -0.4465353190898895f, 0.8045015335083008f, 0.830817461013794f, -0.4116531312465668f, -0.19528378546237946f, 1.0318245887756348f, -0.6419956088066101f, -1.068763256072998f, -0.038084980100393295f, -0.8322976231575012f, 0.8178791999816895f, 0.14809109270572662f, -1.4732903242111206f, -0.10355143249034882f, -0.5564566254615784f, 0.3039886951446533f, 0.9035240411758423f, 0.5980544090270996f, -1.4176050424575806f, -0.0560019314289093f, -1.3687875270843506f, -0.5567623376846313f, -0.36739063262939453f, 1.5230790376663208f, 1.4708421230316162f, -0.4366856515407562f, -1.1750401258468628f, 0.24909113347530365f, 1.8527616262435913f, -1.680544137954712f, 0.659866988658905f, -0.5326191782951355f, 0.5515633225440979f, 1.6462424993515015f, 0.2259528934955597f, 0.27357611060142517f, -0.9556253552436829f, 0.43514177203178406f, -1.1005570888519287f, 0.9130839705467224f, -0.45959728956222534f, 2.1901278495788574f, -0.682377278804779f, -0.22877585887908936f, 0.7370786666870117f, -1.690212607383728f, 0.30514323711395264f, -0.0909213200211525f, -1.1586445569992065f, -0.7481539845466614f, 1.1121810674667358f, -0.4421563148498535f, 1.0719655752182007f, 0.17691805958747864f, -1.2427209615707397f, 0.10128959268331528f, 0.19372011721134186f, 0.24058400094509125f, 0.9057638049125671f, 0.11222076416015625f, -0.059362635016441345f, -2.4078524112701416f, -1.2304182052612305f, -0.026510445401072502f, 0.8861871957778931f, -0.7287083268165588f, 1.380319356918335f, 0.6684188842773438f, -1.7624090909957886f, 0.6292073726654053f, -0.7828742265701294f, -0.08022477477788925f, 0.9383800625801086f, -0.4210285246372223f, -0.49148616194725037f, -0.9306524395942688f, -1.267333745956421f, -0.6902195811271667f, 0.23923389613628387f, 1.4039400815963745f, -2.0282845497131348f, -1.0275602340698242f, 0.9761411547660828f, -0.06743653863668442f, 0.06586836278438568f, -0.7352823615074158f, 0.19450226426124573f, -0.7134578824043274f, 0.16818134486675262f, -1.1888638734817505f, -0.37875133752822876f, -0.321153849363327f, 1.3544011116027832f, -0.423284113407135f, -0.9974878430366516f, -0.9902269244194031f, -0.1406998336315155f, 0.8166614174842834f, 1.0789191722869873f, 1.7252999544143677f, 1.1660224199295044f, 0.5868375301361084f, -0.11877616494894028f, -0.34278497099876404f, -0.08240477740764618f, 0.8542949557304382f, -0.9149259924888611f, 0.5388334393501282f, -1.688096284866333f, -1.05665123462677f, -0.33759868144989014f, -0.2788163125514984f, -0.3783758282661438f, -0.2296314239501953f, -1.1831741333007812f, -1.034787893295288f, -0.8083778023719788f, -0.4777121841907501f, 0.8160690665245056f, 0.8076881766319275f, 0.5698519945144653f, -0.4374706447124481f, -0.7118549942970276f, 2.338996648788452f, -0.0368775799870491f, -0.1774366796016693f, 1.5478016138076782f, 1.0848923921585083f, -0.38529422879219055f, -0.5037281513214111f, 0.9991070628166199f, -0.715931236743927f, 0.28394055366516113f, 0.5939877033233643f, 0.31532949209213257f, 0.27048054337501526f, 0.9632524251937866f, -1.2189302444458008f, -0.06929556280374527f, 0.061022937297821045f, 0.822035551071167f, 0.38992002606391907f, -0.49170032143592834f, -1.689696192741394f, 0.05627693980932236f, -0.2405131459236145f, 0.9160073399543762f, 0.7348390221595764f, -0.6902630925178528f, -0.9186923503875732f, -0.6866548657417297f, -0.10339432954788208f, -0.32694968581199646f, 1.2450132369995117f, 0.6069359183311462f, -0.0937698483467102f, -0.004049783106893301f, 2.6462395191192627f, 1.7450222969055176f, 0.2888743579387665f, 1.1111222505569458f, 1.1171294450759888f, -0.7966887354850769f, 1.0229419469833374f, -1.0742470026016235f, -1.4498995542526245f, -0.5231860280036926f, 0.3438241481781006f, 0.20375612378120422f, -1.4713748693466187f, 0.19777953624725342f, -0.3222813904285431f, 0.5210180282592773f, 1.036444902420044f, 0.2102498859167099f, -1.1502959728240967f, -0.25832024216651917f, 0.2814403772354126f, -0.5159256458282471f, 1.522001028060913f, -0.4228188693523407f, 0.5675636529922485f, 0.47951194643974304f, -0.9628922939300537f, 0.16651970148086548f, 0.17922191321849823f, -0.6149380207061768f, 0.884855329990387f, -0.7095723152160645f, 1.5105708837509155f, 0.9143079519271851f, -1.2950124740600586f, 0.9506115317344666f, -1.3415539264678955f, 0.9584245681762695f, -1.834415316581726f, -0.5945284366607666f, -1.6709949970245361f, -1.3442386388778687f, -0.7921901941299438f, 0.1253465712070465f, -0.9972168207168579f, 0.9316890239715576f, -1.098996639251709f, -0.8827964067459106f, -0.03510960936546326f, 0.33106282353401184f, -1.535096526145935f, 1.5791680812835693f, 1.562730073928833f, 1.0339767932891846f, -0.644834041595459f, -0.7956817150115967f, 0.4747835099697113f, 0.6450149416923523f, -2.4694929122924805f, -1.2703917026519775f, 1.4028544425964355f, 0.39471593499183655f, -0.2914227247238159f, -0.04415971040725708f, 0.6573874354362488f, -1.8288997411727905f, 0.515746533870697f, -2.1813249588012695f, -0.7443293333053589f, 0.4601408839225769f, 1.2225193977355957f, -2.047572374343872f, 1.09447181224823f, 0.42030036449432373f, -0.6366481781005859f, -2.1695642471313477f, 1.1429638862609863f, 0.8955739736557007f, 1.6797635555267334f, -0.6834523677825928f, 0.8514192700386047f, -0.8422428369522095f, -1.0746608972549438f, -0.9010616540908813f, -0.6362974643707275f, -1.087014079093933f, -0.028106562793254852f, -0.1531383991241455f, 1.0988678932189941f, -0.03311381861567497f, -0.892038881778717f, -1.3073756694793701f, 0.14662222564220428f, 0.8670119643211365f, 3.2090775966644287f, -1.3412030935287476f, 0.3424357771873474f, 0.19632267951965332f, -0.1660011112689972f, -0.3333166837692261f, 1.0084987878799438f, 0.9129180312156677f, 0.5135545134544373f, -0.8764104843139648f, -1.4057948589324951f, -0.6906246542930603f, 0.3960992097854614f, -0.2924385964870453f, 0.3751489818096161f, -1.1075292825698853f, 0.37509971857070923f, -1.4790199995040894f, 0.5692160725593567f, -1.146249532699585f, -1.904903769493103f, 0.9906712174415588f, -0.9967307448387146f, -2.3042960166931152f, -1.8701173067092896f, 1.6571006774902344f, -0.5442659258842468f, 0.3686337471008301f, -0.17834435403347015f, -0.7094868421554565f, 0.24787208437919617f, 0.7445037364959717f, 1.5205289125442505f, -1.3338558673858643f, -2.0718472003936768f, -1.8923321962356567f, 2.419602632522583f, -0.24423770606517792f, -0.2663561701774597f, -0.6755290031433105f, -1.124692440032959f, 1.8602604866027832f, 0.6256695985794067f, 0.28849464654922485f, 0.8962987065315247f, 0.8840014338493347f, 0.4107360541820526f, -1.6067783832550049f, 1.4667541980743408f, -1.643362045288086f, -0.7528584599494934f, -0.8185959458351135f, 0.02291426621377468f, -0.7729488015174866f, -0.039141006767749786f, 0.30586403608322144f, -0.2691587805747986f, -0.21813122928142548f, 0.3616786003112793f, -1.017099380493164f, 0.7204318046569824f, -0.10523297637701035f, 1.8706086874008179f, 0.8992230892181396f, -0.23518913984298706f, -0.6936088800430298f, 0.38842713832855225f, -0.9864745140075684f, 0.24743980169296265f, 1.3209456205368042f, 0.5116807222366333f, 0.20717298984527588f, 0.423617959022522f, 0.8553113341331482f, -0.7523386478424072f, -0.7315530180931091f, -0.34988582134246826f, 0.14683414995670319f, 0.337516725063324f, -0.5604003667831421f, -0.3334929347038269f, 0.46565955877304077f, -0.831680178642273f, -0.8464822769165039f, -0.38647857308387756f, -0.1513155996799469f, 0.8186390399932861f, -0.25432348251342773f, -0.47535333037376404f, -0.780207633972168f, 0.778557300567627f, 0.0986286848783493f, -0.754118800163269f, 0.9054679274559021f, -1.1227487325668335f, 0.2190994918346405f, -0.09212681651115417f, 1.8318346738815308f, -0.5303875803947449f, -0.8198762536048889f, -0.8232145309448242f, 0.5506033301353455f, -1.1416276693344116f, 1.8735969066619873f, 1.3694992065429688f, -1.257182240486145f, -1.6220338344573975f, -0.5663302540779114f, 0.6658836007118225f, 0.25084203481674194f, 0.7626033425331116f, -0.9630534052848816f, -0.5952125191688538f, -0.5548334121704102f, 1.2073006629943848f, -0.726753830909729f, -0.25774624943733215f, 0.7335841059684753f, -0.6472665667533875f, 0.4566407799720764f, -0.506052553653717f, -0.07941094785928726f, 0.226389080286026f, 1.392283320426941f, -0.1649594008922577f, -0.8620851635932922f, 1.041149377822876f, -0.18962401151657104f, 2.0505990982055664f, -2.2949159145355225f, -0.8339797258377075f, 2.1567959785461426f, 1.0796829462051392f, -0.815654456615448f, 0.7114473581314087f, -0.24212278425693512f, 0.9910399317741394f, 0.551307737827301f, 0.6295360326766968f, 0.1759207546710968f, 1.6339221000671387f, 1.2599411010742188f, -0.6335592269897461f, -0.8656108379364014f, 0.2853967845439911f, -1.4132448434829712f, 1.7862261533737183f, -0.18690960109233856f, 0.45671215653419495f, 0.5091333985328674f, 0.12447594106197357f, -0.7764351963996887f, -0.9075336456298828f, 1.6591986417770386f, -0.9774589538574219f, -1.437660813331604f, -1.0792182683944702f, -1.0316091775894165f, 0.4912571609020233f, -0.9918735027313232f, 0.37739869952201843f, -0.09308940172195435f, -0.3923634886741638f, -1.9273265600204468f, -0.9188506603240967f, 1.127413034439087f, 1.4187533855438232f, -0.30123433470726013f, -1.6901737451553345f, 0.9281692504882812f, -1.123509168624878f, -1.0524919033050537f, -0.4467827379703522f, 0.523188054561615f, -0.3387533724308014f, -1.7577482461929321f, -0.6524283289909363f, -0.24884779751300812f, -1.2434697151184082f, 1.6212447881698608f, 0.15123429894447327f, 1.4580625295639038f, 0.8593719601631165f, 0.4116728901863098f, -0.3829646706581116f, 0.20710650086402893f, -0.845027506351471f, 0.2616454064846039f, -0.790607750415802f, -0.35345759987831116f, -0.4279586672782898f, 1.68895423412323f, 1.2128044366836548f, 0.9204495549201965f, 0.047616202384233475f, 0.7502005696296692f, -1.6819013357162476f, -0.036870624870061874f, -0.05854296684265137f, 0.6686491370201111f, -0.4365732967853546f, -2.3380324840545654f, 1.8583958148956299f, -1.7244455814361572f, -0.2750453054904938f, 0.4532167315483093f, 1.2953895330429077f, 0.058747299015522f, -0.8122621774673462f, 1.9493451118469238f, 0.7995834350585938f, -0.010960266925394535f, -0.13873475790023804f, -1.9486329555511475f, -0.7052865028381348f, -0.9286441802978516f, 0.5382292866706848f, -1.931458592414856f, 0.17587143182754517f, 1.2172441482543945f, 0.9002020955085754f, 1.1507850885391235f, 0.027491796761751175f, -1.4959322214126587f, -0.18092377483844757f, 0.905379056930542f, 1.720672845840454f, 0.3225809633731842f, 0.5499839782714844f, -0.6421884298324585f, 0.22866190969944f, -0.5370370149612427f, 0.5335260629653931f, -1.0596036911010742f, -2.2213568687438965f, 0.7067712545394897f, -0.21388542652130127f, -0.4895268678665161f, 0.5013054013252258f, -0.9222452044487f, 0.15941572189331055f, -1.377753734588623f, 1.15412437915802f, -1.092998743057251f, -0.8713886141777039f, 0.22347310185432434f, 0.2528358995914459f, 0.21688957512378693f, 0.4699849486351013f, 0.3267545700073242f, -0.09539508819580078f, -0.18269990384578705f, 0.3382812440395355f, -0.6900221109390259f, 0.047023314982652664f, 0.5487117171287537f, -1.9950600862503052f, -0.1581975668668747f, -0.1996922492980957f, 0.46645262837409973f, 0.5603392124176025f, 0.9167394638061523f, -1.983751893043518f, -0.015853818506002426f, 0.033756233751773834f, -1.4661213159561157f, -0.32681238651275635f, -0.5929558873176575f, 0.8652521371841431f, 0.4256573021411896f, -0.2335444986820221f, -0.7175306677818298f, -2.244840621948242f, 1.9783128499984741f, 0.3803824186325073f, 0.04183865338563919f, 0.284446656703949f, -2.161471128463745f, -0.5391823053359985f, 0.3041086494922638f, -0.3503345549106598f, 1.171104907989502f, -0.8915714621543884f, 1.7047383785247803f, -1.7130810022354126f, 1.0918960571289062f, 1.9015947580337524f, -0.2993949353694916f, 0.2431759536266327f, -0.06963914632797241f, -1.9186826944351196f, -1.1858599185943604f, 1.3959892988204956f, 0.31790241599082947f, 1.0485270023345947f, -0.8889532089233398f, -0.1494847685098648f, 0.011521455831825733f, -0.3648742437362671f, -0.14563268423080444f, 1.1415576934814453f, 2.0192317962646484f, -1.1988294124603271f, -0.08969400823116302f, 1.01350998878479f, -0.148199200630188f, -0.20676261186599731f, -0.4151966869831085f, -0.5613080859184265f, -1.5219647884368896f, -0.6867799758911133f, -1.207437515258789f, 0.27414625883102417f, 1.5931971073150635f, 0.8239222168922424f, 0.17494884133338928f, 0.37811675667762756f, -0.5250972509384155f, 1.1867752075195312f, 0.44894057512283325f, 0.6020882725715637f, 0.27788910269737244f, 0.3914108872413635f, -0.669601321220398f, -0.08302459120750427f, 0.06917848438024521f, -0.263455331325531f, 0.7421362996101379f, 0.2899683713912964f, -0.4380805194377899f, -1.3862485885620117f, -0.7712535858154297f, -0.3168620467185974f, -1.6128871440887451f, -0.9478936195373535f, -0.37378421425819397f, -0.8723973631858826f, 0.43149101734161377f, -0.7728766202926636f, -0.8030177354812622f, 0.46446937322616577f, -0.17089581489562988f, 2.706796407699585f, 0.6624157428741455f, -0.654021143913269f, 0.7278003692626953f, 0.09257330745458603f, -0.1797974407672882f, 0.7003864645957947f, -1.250577688217163f, 0.9090378284454346f, -0.15779435634613037f, -0.43905171751976013f, -0.16366083920001984f, -1.2389750480651855f, -1.0557494163513184f, 1.2461082935333252f, -0.7245869040489197f, -1.044531226158142f, 0.5626670122146606f, 0.17291614413261414f, 1.9904685020446777f, -1.181625247001648f, 0.30392396450042725f, 2.225896120071411f, 0.30583375692367554f, -0.6297805309295654f, 0.7762312889099121f, -0.5106104016304016f, -0.5062925219535828f, 0.7492969632148743f, 1.0681610107421875f, -2.636631488800049f, 0.6954406499862671f, -0.32303762435913086f, 0.5969774723052979f, -0.6247759461402893f, 1.647339940071106f, -1.7078118324279785f, 0.09625459462404251f, -0.28645211458206177f, 1.0167992115020752f, -0.6579540371894836f, -0.3999125063419342f, -1.7973577976226807f, -0.9080927968025208f, 1.1592087745666504f, -1.467360258102417f, 1.7191014289855957f, -0.3673190176486969f, 2.3575830459594727f, -0.459722101688385f, -0.058640673756599426f, -1.2418878078460693f, -0.023634955286979675f, -2.0428922176361084f, 0.19053655862808228f, 0.5756992697715759f, -0.5177616477012634f, 0.9197074770927429f, 0.9861821532249451f, 0.9352383017539978f, 0.30303436517715454f, -0.5343928933143616f, -0.2886335253715515f, 0.9647671580314636f, 1.7285969257354736f, -0.8043005466461182f, -0.28731220960617065f, -1.2584627866744995f, 0.08387812972068787f, 0.9973886609077454f, -0.7661278247833252f, -0.3684079647064209f, -0.16042770445346832f, 0.731488823890686f, -0.4989534318447113f, 0.5183284878730774f, 0.6223585605621338f, 0.017574317753314972f, 1.3758805990219116f, -0.024098770692944527f, -0.11441737413406372f, -0.2840602993965149f, -0.10638472437858582f, 0.3422453701496124f, -0.21973253786563873f, -0.09440521895885468f, -0.514839768409729f, 0.5137941241264343f, 0.3945278227329254f, 0.5612776279449463f, -0.006535662803798914f, 0.8489260077476501f, 0.4735972583293915f, 0.5418957471847534f, 0.0022529142443090677f, -0.5435486435890198f, 0.18075302243232727f, -0.7696079015731812f, -1.0346392393112183f, 0.7314628958702087f, 1.6726516485214233f, 0.35546284914016724f, 0.30347952246665955f, 0.04806268960237503f, 0.5032230019569397f, -0.025565603747963905f, -1.8534711599349976f, 1.0129450559616089f, -0.07218152284622192f, 0.03116939589381218f, -0.755984902381897f, -0.7956190705299377f, -0.6868169903755188f, 1.9091075658798218f, -0.4942598342895508f, -0.06308790296316147f, 0.9503589868545532f, 0.6692175269126892f, 0.12500713765621185f, 0.5085462331771851f, -1.0646655559539795f, -0.4647725224494934f, -0.9152674674987793f, 0.7144214510917664f, 1.472544550895691f, -1.7723859548568726f, 0.0483609139919281f, 0.20343580842018127f, -0.6890958547592163f, -0.5412123203277588f, 2.0465071201324463f, -0.5884720087051392f, -0.8352398872375488f, 0.0502433180809021f, 0.4375612735748291f, 1.3833050727844238f, -1.3894257545471191f, 0.3897654116153717f, -1.5383917093276978f, 1.8197290897369385f, -0.5204964280128479f, 1.4799479246139526f, 0.4248008131980896f, -0.30355918407440186f, 0.07046826928853989f, -1.5127924680709839f, 1.3581115007400513f, 0.32402706146240234f, -1.0680902004241943f, 0.08959023654460907f, 3.1311562061309814f, -0.8544708490371704f, -0.8357695937156677f, 1.2985942363739014f, -0.7110929489135742f, -0.9653112292289734f, -1.340206503868103f, 0.03399703651666641f, -0.4458200931549072f, 0.151767760515213f, 1.6231015920639038f, -0.429679274559021f, -0.16159531474113464f, -1.0276970863342285f, -0.9918712973594666f, 0.29410406947135925f, -0.3595946729183197f, -0.13289135694503784f, -0.09766664355993271f, -1.9935065507888794f, 0.11608057469129562f, 0.9187806844711304f, 0.12344544380903244f, 2.8273682594299316f, 0.47407886385917664f, -1.9149128198623657f, -1.5384353399276733f, -0.6529980897903442f, 0.16414928436279297f, -0.19700200855731964f, 1.038715124130249f, -0.3908529281616211f, 1.0207839012145996f, -1.2268577814102173f, -2.0579991340637207f, 0.5987722277641296f, -0.13595744967460632f, 0.24731674790382385f, -2.150991439819336f, 1.2150318622589111f, -0.33008328080177307f, -0.5156939625740051f, 0.022917015478014946f, -0.6746217608451843f, 1.309282898902893f, -1.6657313108444214f, 0.32850465178489685f, -0.8491258025169373f, 0.9747982025146484f, -0.37322181463241577f, 0.6090918779373169f, -0.7589855194091797f, -0.6492360830307007f, -1.2674946784973145f, -0.382905513048172f, 0.8315308690071106f, 1.8437397480010986f, 0.021685972809791565f, 0.28762421011924744f, 0.39912644028663635f, 2.4587013721466064f, 0.4567311406135559f, 1.106606125831604f, 0.9169553518295288f, -0.23127765953540802f, -1.498667597770691f, 0.6427391171455383f, 0.18222902715206146f, -1.0757185220718384f, 1.345549464225769f, 0.32771772146224976f, 1.3408823013305664f, 1.2158557176589966f, 0.95885169506073f, 0.5136737823486328f, 0.5763065814971924f, 0.0778578370809555f, -1.061161994934082f, 2.042306900024414f, 0.6509259343147278f, -1.0072094202041626f, 0.35781362652778625f, -1.0799492597579956f, 0.04710597172379494f, 2.201401472091675f, 0.2908516526222229f, -0.08251824229955673f, 1.5195056200027466f, -0.3584267199039459f, -0.6102548837661743f, -1.5060930252075195f, -0.7732543349266052f, 1.0847911834716797f, 1.4872429370880127f, 0.6041548848152161f, -1.1257244348526f, 0.3761575222015381f, 1.959293246269226f, -0.7958163619041443f, 0.4539487659931183f, -2.5069937705993652f, -1.361000418663025f, 1.183761477470398f, -1.1673825979232788f, -0.25077250599861145f, 0.33264845609664917f, 1.1230510473251343f, 1.8761743307113647f, 0.273639976978302f, 0.2801147401332855f, 0.7281787991523743f, 0.02403142862021923f, 2.3937904834747314f, -1.3129955530166626f, -0.48624828457832336f, 0.4012764096260071f, 1.0767757892608643f, 1.1279785633087158f, 0.02586616761982441f, -0.14947263896465302f, 0.39506977796554565f, 1.9742354154586792f, -0.4404173791408539f, 2.3815414905548096f, -0.34589144587516785f, 2.694831609725952f, -0.8544124364852905f, -0.27781322598457336f, 0.7772916555404663f, -0.43923330307006836f, 1.1729096174240112f, 1.397361159324646f, -0.41255858540534973f, -0.4339471161365509f, 0.9574882388114929f, 1.254128336906433f, -1.0384563207626343f, 0.5251074433326721f, -0.8383287787437439f, 1.7227904796600342f, -1.1731188297271729f, 0.7272883057594299f, -1.7194626331329346f, 2.383774995803833f, -1.1649107933044434f, -0.20503655076026917f, -0.5375787019729614f, 1.2967820167541504f, -1.6138769388198853f, 0.7535606026649475f, -0.38888975977897644f, 2.5921998023986816f, 0.08543383330106735f, -2.0308878421783447f, 0.6461803913116455f, 1.217452883720398f, 0.03121299482882023f, 1.082465648651123f, -1.8998514413833618f, -2.7074427604675293f, -0.7867730259895325f, -0.6195639967918396f, 1.8281400203704834f, 0.4877544343471527f, 0.1366790533065796f, -0.5910199284553528f, 2.3618080615997314f, 1.2076122760772705f, -0.5487976670265198f, 1.5311497449874878f, 0.44362834095954895f, -2.4579336643218994f, 0.6496618390083313f, 0.02449978142976761f, 0.25982725620269775f, 0.7348682284355164f, -0.006029692944139242f, -0.7884382605552673f, 1.1371253728866577f, -1.7366209030151367f, 0.9614791870117188f, -1.3104760646820068f, -1.4218922853469849f, -0.3201002776622772f, 0.011662798002362251f, 0.7528783082962036f, -0.8849666118621826f, 0.8982152342796326f, -1.6744109392166138f, -1.1447242498397827f, 0.22346419095993042f, -0.9369489550590515f, -0.17544429004192352f, 0.8354768753051758f, -1.2516489028930664f, 0.26434803009033203f, 1.2949540615081787f, -0.5020677447319031f, 0.45930108428001404f, -1.3068112134933472f, 0.9302626848220825f, 1.0709624290466309f, 1.3384746313095093f, 0.27829068899154663f, -1.328766107559204f, -0.2891634404659271f, -2.518200635910034f, -2.533468723297119f, -0.022988365963101387f, -0.5563327670097351f, 2.0752601623535156f, -0.8403553366661072f, 0.9517558813095093f, -2.1648738384246826f, 0.9794597625732422f, 0.38047143816947937f, -1.9277540445327759f, 0.9197121858596802f, -1.6067678928375244f, -1.3559657335281372f, -0.25936615467071533f, 0.0861901044845581f, -0.37659966945648193f, 0.9756653904914856f, -0.8411983847618103f, -0.9621247053146362f, -0.5808306932449341f, 1.1524169445037842f, -0.5585207939147949f, -0.13208438456058502f, -1.0929938554763794f, -2.4860122203826904f, -0.038849566131830215f, 0.8021038770675659f, -0.18547378480434418f, 0.534229576587677f, -0.49810588359832764f, 0.8963930010795593f, -1.2691730260849f, 0.042280785739421844f, 0.9680984020233154f, -1.7608284950256348f, 1.0980932712554932f, 0.8579375743865967f, -0.9562888145446777f, -0.7236151099205017f, 0.2793406844139099f, -0.8662601113319397f, 0.8459339737892151f, 0.6116387844085693f, -0.8089803457260132f, -0.20030127465724945f, -2.4035587310791016f, 1.5633418560028076f, -0.6294617652893066f, 0.8042920231819153f, 0.6676139235496521f, -0.5181459784507751f, 0.9539962410926819f, 0.7256157398223877f, -0.5767995715141296f, -0.01296310406178236f, 1.34257972240448f, 0.8595607280731201f, 0.24403470754623413f, 1.9769524335861206f, 0.5327519178390503f, -0.8788067102432251f, -0.2609003186225891f, -0.1277228742837906f, -0.38701093196868896f, -0.10875839740037918f, 0.7135027647018433f, 0.037005677819252014f, 0.7367677688598633f, -1.073968529701233f, -0.8155962824821472f, 0.3688916862010956f, 0.5232675075531006f, -1.1358050107955933f, 2.01452898979187f, -0.5778390765190125f, -0.2187574803829193f, 0.35567042231559753f, 0.7510744333267212f, 0.2171924114227295f, 0.4019394814968109f, 0.9250243306159973f, -0.24931922554969788f, -0.3473069369792938f, 0.8122217059135437f, -0.6078834533691406f, -0.8096639513969421f, -0.4163765013217926f, -0.9129456877708435f, -1.0231975317001343f, 0.15973883867263794f, 1.5596561431884766f, 0.2012278288602829f, 0.9135115742683411f, -0.7519399523735046f, 2.7958858013153076f, 0.41675102710723877f, 0.0033115341793745756f, -1.398554801940918f, 1.5899420976638794f, -0.7806249856948853f, -0.7495994567871094f, 0.2814400792121887f, -0.08736804872751236f, -1.1191902160644531f, -0.4140253961086273f, 0.2933676540851593f, -0.016588633880019188f, 0.4729222357273102f, 1.0479241609573364f, 0.32106152176856995f, 1.162279486656189f, 1.5198776721954346f, 1.134521245956421f, 0.013782661408185959f, 1.5933806896209717f, 0.11106725037097931f, 0.5933053493499756f, 0.0023419519420713186f, 0.9275448322296143f, 0.1871725618839264f, 1.0549991130828857f, -0.04648401960730553f, 1.6230204105377197f, 0.9345911145210266f, 0.6804553866386414f, 2.2367258071899414f, -0.6858885884284973f, -1.308877944946289f, -1.208367109298706f, -1.4461109638214111f, -0.6192750930786133f, -0.40478718280792236f, 0.45292869210243225f, 0.40945965051651f, 0.9656304121017456f, 0.4625106155872345f, -1.4208135604858398f, 0.2507438659667969f, -0.5702037215232849f, -0.05394743010401726f, -0.7306843400001526f, -1.0346637964248657f, 0.7631427049636841f, 0.9210829138755798f, -0.6593214273452759f, 0.24809309840202332f, 0.7311660647392273f, 2.3288936614990234f, -1.06203031539917f, -0.5385692119598389f, 0.4127616584300995f, 0.3390391767024994f, 1.7734715938568115f, 1.655669927597046f, -0.08070362359285355f, -0.303115576505661f, 0.3240552544593811f, 1.0401465892791748f, 0.5915576815605164f, -1.4803826808929443f, -0.12040789425373077f, 0.16437974572181702f, 0.7806977033615112f, 1.228264331817627f, 0.4453698992729187f, -0.3023916184902191f, -0.7796450257301331f, 1.15239417552948f, -0.4312492907047272f, 0.4804547131061554f, -0.0037577631883323193f, -0.23722761869430542f, 0.1753881573677063f, -0.3416663408279419f, -0.4271427094936371f, 1.1991220712661743f, 0.8689031600952148f, -0.6073640584945679f, 0.11616495996713638f, 1.9142365455627441f, -1.1450425386428833f, 1.60905921459198f, -0.5139697194099426f, -0.3140712380409241f, -0.8430630564689636f, -1.2868314981460571f, 0.5520419478416443f, 2.584942102432251f, 0.4215601086616516f, 2.558887481689453f, -1.2483208179473877f, -0.9826495051383972f, 0.7373914122581482f, 1.4831291437149048f, 0.6203920245170593f, 0.33499792218208313f, -0.5796453952789307f, -0.24162966012954712f, 1.4769864082336426f, -1.1677722930908203f, 0.43188926577568054f, 0.7582098245620728f, 0.26907673478126526f, -0.16138136386871338f, 1.1750508546829224f, 0.5351220369338989f, 1.225508689880371f, 0.9085782766342163f, 0.16776901483535767f, -0.009964917786419392f, -0.9104782342910767f, 0.3974350690841675f, -0.5543625354766846f, 1.281534194946289f, 0.38322561979293823f, 1.3406338691711426f, -0.43185070157051086f, -1.897344708442688f, -2.652250289916992f, -0.43282368779182434f, 0.932184100151062f, 0.4349152445793152f, -0.7768413424491882f, -0.19172848761081696f, 1.1136181354522705f, -1.214506983757019f, 2.1966660022735596f, 0.41227126121520996f, -0.7043505311012268f, 0.09847358614206314f, 1.8926005363464355f, 0.6903649568557739f, 1.4352598190307617f, 1.0864993333816528f, -0.3989373445510864f, -2.0487279891967773f, -1.1373023986816406f, 0.9046337604522705f, 0.5279921293258667f, -0.5005130171775818f, 0.0378837063908577f, -1.0129538774490356f, 0.7843693494796753f, 1.3280545473098755f, -0.5138816833496094f, -0.04832516983151436f, -0.8721303939819336f, -1.3769567012786865f, -1.8487751483917236f, 0.30561649799346924f, -0.42448726296424866f, 0.6269863247871399f, -0.6438542604446411f, 3.0598578453063965f, 0.8688741326332092f, 0.7331935167312622f, 1.7552188634872437f, -0.24250611662864685f, 1.9880743026733398f, -1.1370364427566528f, -2.2499585151672363f, -0.5810695886611938f, 0.8816671967506409f, -2.0325005054473877f, -0.35224223136901855f, 0.1006089523434639f, 1.5255436897277832f, -1.6215003728866577f, 0.5298449397087097f, -1.3718810081481934f, -0.6298099756240845f, -0.65668123960495f, 0.4244276285171509f, -0.40122607350349426f, -0.4890124797821045f, 0.9571828246116638f, -0.8436367511749268f, -0.3680597245693207f, -0.47244831919670105f, -0.6993737816810608f, 0.4523862302303314f, -1.090578317642212f, 0.9411556720733643f, -1.931666612625122f, 0.1285097301006317f, -1.0134315490722656f, 0.7479439973831177f, 0.20112790167331696f, -0.8995627760887146f, -0.2906798720359802f, 0.1047126054763794f, -0.5943686366081238f, -1.727795124053955f, 1.0557186603546143f, -0.5037193298339844f, -1.5121967792510986f, -0.14455023407936096f, -0.7709440588951111f, 1.3850905895233154f, 0.06768468767404556f, -2.205592393875122f, -0.9360390901565552f, 1.6600853204727173f, -0.551665186882019f, -0.3103943169116974f, -0.6468674540519714f, -0.22681531310081482f, -0.3377947509288788f, 0.6926270723342896f, -0.5544835925102234f, -0.10487986356019974f, -0.12251695245504379f, -0.10804381966590881f, -1.4534186124801636f, -0.5115614533424377f, 0.9907002449035645f, 0.7906493544578552f, -0.34170281887054443f, -1.8583146333694458f, 0.6896669268608093f, 0.8880993127822876f, 1.2050648927688599f, -0.5951094031333923f, -0.3993092179298401f, -0.30013880133628845f, -0.014673005789518356f, 0.6156877279281616f, -0.9164139032363892f, -0.06423450261354446f, -0.33351314067840576f, 0.5679208636283875f, 0.5063413977622986f, 1.8924111127853394f, -0.7531763911247253f, 1.0589748620986938f, -0.010384922847151756f, 0.41770482063293457f, 0.16313597559928894f, -0.26724591851234436f, 0.1958005577325821f, 0.8279790282249451f, -0.7302268147468567f, 0.4149245321750641f, 2.0801644325256348f, 0.1493149846792221f, -0.4159306585788727f, 1.2188113927841187f, 0.8605815768241882f, 0.8988009691238403f, -0.4320070743560791f, -0.011584565043449402f, 0.1905190795660019f, 0.45351630449295044f, -0.2722189724445343f, 0.4377744197845459f, 0.6674123406410217f, -0.8114345669746399f, -1.1677125692367554f, 1.4608654975891113f, -1.6388529539108276f, 0.9854339957237244f, 0.01134670153260231f, 0.3140886127948761f, 1.081308126449585f, 0.1636349856853485f, -1.8501884937286377f, -0.6727942824363708f, -1.8205945491790771f, -0.16954289376735687f, -0.974631667137146f, -0.5593003630638123f, -0.2645290195941925f, 0.2448410987854004f, 0.4654839038848877f, 1.1221051216125488f, -1.2755296230316162f, 1.968849539756775f, 1.1931818723678589f, -0.9534568786621094f, 0.6892605423927307f, -0.03748741000890732f, -1.3567429780960083f, -0.930943489074707f, -0.4104326069355011f, 1.6298463344573975f, -0.3964575529098511f, -0.8092941045761108f, -0.8157675862312317f, -0.1790880262851715f, -0.7333499789237976f, 2.0713958740234375f, 0.26833969354629517f, -0.4909171462059021f, 0.0867118388414383f, -0.19213901460170746f, -0.020139964297413826f, 1.3420833349227905f, 0.8771296143531799f, 0.9378947615623474f, 0.023886969313025475f, 1.5275574922561646f, -0.08036366105079651f, -0.9117053151130676f, -1.1944957971572876f, 0.9197462201118469f, 0.28947049379348755f, 0.5755492448806763f, -1.0247280597686768f, 0.19418835639953613f, 1.5901795625686646f, -0.7810332179069519f, 0.18075843155384064f, 0.5682910680770874f, -1.3145383596420288f, -0.037264980375766754f, -1.1747896671295166f, -0.5241218209266663f, 0.8552600145339966f, 0.7151663303375244f, 0.854972243309021f, 0.6970289349555969f, -0.7869400978088379f, -0.22646142542362213f, 0.8945121765136719f, -0.07112658768892288f, 0.19750215113162994f, -0.08805552124977112f, 0.46580448746681213f, 0.745457112789154f, -0.6745474338531494f, -0.8672813177108765f, -2.0061123371124268f, 2.204427480697632f, -0.336713045835495f, -0.07664714008569717f, 0.6008157134056091f, 0.7986276745796204f, -1.3824840784072876f, -0.3528830111026764f, 0.9141533374786377f, 0.5920729637145996f, 0.7283052206039429f, 0.5198004245758057f, 0.40039631724357605f, -0.16163653135299683f, 0.3812867999076843f, 0.2818247973918915f, -0.259854257106781f, -0.17636136710643768f, 1.2899055480957031f, -1.809929609298706f, 0.04499081149697304f, -0.5114196538925171f, 1.6922872066497803f, -0.5404288172721863f, -0.06902796775102615f, -0.900188148021698f, 0.16361606121063232f, -0.5697844624519348f, -0.17199715971946716f, 1.9494951963424683f, -1.5725066661834717f, -0.11967476457357407f, 0.6768812537193298f, -2.550161600112915f, 1.3886760473251343f, 2.071373224258423f, -0.4157007336616516f, 1.7397890090942383f, -1.5081530809402466f, -0.9471526145935059f, 0.4156656265258789f, -0.7409192323684692f, -2.0307180881500244f, 0.7459039092063904f, -0.23346112668514252f, -1.7168304920196533f, -1.5648266077041626f, 1.1608803272247314f, -1.7281652688980103f, 0.6240199208259583f, 0.19563332200050354f, 0.9567276835441589f, 0.9916862845420837f, -0.7535449862480164f, 1.3851630687713623f, 0.8076578974723816f, 0.4444909691810608f, 0.12085723131895065f, 0.794389545917511f, -0.2691969871520996f, 1.6320013999938965f, 1.4558833837509155f, 0.8513883352279663f, 1.305290937423706f, 0.8631715178489685f, -0.5379660725593567f, 0.5391438603401184f, -0.18788452446460724f, -1.6509469747543335f, 0.6196591854095459f, -1.1270289421081543f, -0.08517079800367355f, 1.0406969785690308f, -0.5241701006889343f, -0.5131880044937134f, 0.4295358955860138f, 1.8150509595870972f, -1.150861382484436f, -0.30584630370140076f, 0.8076725602149963f, 0.7956660985946655f, 0.2235465794801712f, -0.05132210999727249f, 0.7380313277244568f, 1.5020791292190552f, 0.8205595016479492f, -0.056694693863391876f, -0.17149198055267334f, 0.9732901453971863f, -0.9835246801376343f, 0.1294081211090088f, -1.1072022914886475f, -0.1295740306377411f, 1.005579948425293f, -1.3748044967651367f, 1.197999119758606f, -2.2044677734375f, -1.1458145380020142f, -1.4537255764007568f, -0.2384955883026123f, -1.6215006113052368f, -0.3137211501598358f, -0.7217759490013123f, -1.6709321737289429f, 0.3768741488456726f, -1.8265146017074585f, 0.48899734020233154f, -0.5580688714981079f, 0.37294480204582214f, -0.6878536939620972f, -1.4576401710510254f, -0.6108043789863586f, -0.6761788725852966f, -1.9144877195358276f, 0.9340372085571289f, -0.4341937303543091f, 1.4776188135147095f, 0.7693286538124084f, 2.331334352493286f, -0.6489352583885193f, 0.4903559982776642f, -0.2804304361343384f, 0.941003680229187f, -0.11338216066360474f, -0.8615462183952332f, 0.13828350603580475f, 0.013346405699849129f, 0.33505237102508545f, -0.24309676885604858f, -0.6505504250526428f, -0.33003056049346924f, 1.3748613595962524f, -1.033202886581421f, 0.05953902378678322f, -0.2076457291841507f, -0.9487320184707642f, -0.2869058847427368f, -0.902455747127533f, -1.5298161506652832f, -1.2186866998672485f, 1.1458920240402222f, 0.20151416957378387f, -0.4259151816368103f, -0.1506912112236023f, -2.106266498565674f, -0.6301612257957458f, 1.2424033880233765f, -0.025852564722299576f, -0.540851891040802f, -0.3480299413204193f, -0.7818117141723633f, 0.5524802803993225f, 0.8857057690620422f, -1.3390332460403442f, -0.40026965737342834f, 0.7468986511230469f, -0.3904872536659241f, 0.7475174069404602f, -0.4455646276473999f, 1.0798193216323853f, 2.6205930709838867f, 0.6864228844642639f, 1.5354714393615723f, -0.1950068324804306f, -1.4005751609802246f, 0.4377126097679138f, -0.05925627797842026f, -0.552335798740387f, -0.7325940728187561f, 0.5038686990737915f, -0.18185997009277344f, -0.7621927261352539f, -0.8477222323417664f, 0.6641071438789368f, 2.3913745880126953f, 0.4669826030731201f, 0.7512773275375366f, -0.3845667541027069f, -0.06630031019449234f, 0.6945987939834595f, -0.7470712065696716f, 0.2076980471611023f, -0.12689319252967834f, -0.5187065005302429f, 0.8009016513824463f, -0.7448427081108093f, 1.9110214710235596f, 0.3227183520793915f, -0.5433307886123657f, -2.0223824977874756f, 0.7815577387809753f, 1.8362489938735962f, -0.47440794110298157f, -0.38593149185180664f, 0.36727604269981384f, 0.344801127910614f, 0.5554131865501404f, 0.35217124223709106f, 0.7673863768577576f, 0.08270980417728424f, 0.3816434144973755f, -1.0023962259292603f, -0.22075708210468292f, -0.6525165438652039f, -2.485137939453125f, 0.614276111125946f, 0.9891844391822815f, -1.6560537815093994f, 0.17567120492458344f, 0.8857656121253967f, 0.022398322820663452f, -0.24084077775478363f, 1.4238216876983643f, 0.8160249590873718f, 0.2655353844165802f, 0.744394838809967f, -0.17409075796604156f, -0.9177995324134827f, -2.1325347423553467f, -0.4131704568862915f, -0.4764634966850281f, 0.5031798481941223f, 0.27043330669403076f, -0.04328599572181702f, 1.035592794418335f, 1.341705560684204f, -0.11890244483947754f, -0.07687754184007645f, 0.5947466492652893f, 0.5723721385002136f, 2.663830518722534f, 0.7388755679130554f, -0.46826601028442383f, -1.59627366065979f, -1.6671663522720337f, 0.3388274610042572f, 0.3750116229057312f, -1.3291982412338257f, 0.5636889338493347f, 0.8051766753196716f, 0.7449150681495667f, -0.1739114224910736f, 1.1078135967254639f, 0.5147720575332642f, 0.8934884667396545f, -1.5113967657089233f, -0.8514725565910339f, 0.3001462519168854f, 0.17312221229076385f, 0.08587908744812012f, 0.10766935348510742f, 1.0629868507385254f, 1.841042399406433f, -0.5686787366867065f, -1.44584059715271f, -0.2561878263950348f, 1.1728384494781494f, -0.428275465965271f, -0.2985764443874359f, 0.14890146255493164f, 1.661197304725647f, -2.622263193130493f, -0.43193310499191284f, -0.3783835768699646f, 0.23239530622959137f, -0.7321792840957642f, -0.40373992919921875f, 1.2062691450119019f, -0.6938396096229553f, -1.5164529085159302f, 2.0242106914520264f, -0.8075132966041565f, 0.09412498027086258f, 0.27298247814178467f, 0.5441054701805115f, -2.4758286476135254f, 0.21060240268707275f, 0.2536042630672455f, -0.7318422794342041f, 0.05695872753858566f, 0.15881912410259247f, 0.6510986685752869f, 0.020065616816282272f, 0.8478670120239258f, 2.0752015113830566f, -0.38090255856513977f, -0.9460003972053528f, -0.4590383470058441f, -0.6942368745803833f, 0.30009862780570984f, -1.389849305152893f, 1.2990328073501587f, -1.8040062189102173f, 1.837660551071167f, -0.7219073176383972f, -0.3056040108203888f, 0.9780712127685547f, 0.9052547812461853f, -0.901805579662323f, -0.2280527949333191f, -0.948022723197937f, -0.15365807712078094f, 1.032126545906067f, 0.17955327033996582f, -0.2721782922744751f, 0.15856090188026428f, 1.1139295101165771f, -0.5713488459587097f, -0.9870969653129578f, -1.6782877445220947f, 1.8284801244735718f, -0.4982566237449646f, 0.47082117199897766f, -0.591300904750824f, -0.41513174772262573f, 0.5321887731552124f, 1.317676305770874f, -0.7898051142692566f, -0.5088178515434265f, 0.23661386966705322f, 1.2172428369522095f, 0.2148703932762146f, 1.2481820583343506f, 2.4610743522644043f, 0.14153295755386353f, 1.837676763534546f, -0.2276618480682373f, 0.2129386067390442f, 0.8061029314994812f, 0.7021245956420898f, -0.6485925316810608f, -0.47295162081718445f, 2.2190706729888916f, -1.868390679359436f, -1.3732751607894897f, -1.013265609741211f, 1.365964651107788f, 1.1893959045410156f, -2.7800850868225098f, -1.0310310125350952f, -0.702293336391449f, 0.8099242448806763f, 0.8948581218719482f, -0.40090087056159973f, -0.08886078000068665f, -0.9508647918701172f, 0.28014296293258667f, 0.06467004120349884f, -2.057403087615967f, 1.1129963397979736f, -0.7580298185348511f, 0.0977560505270958f, 0.8891482949256897f, 0.3460270166397095f, -0.11394428461790085f, 1.4566701650619507f, -1.1309462785720825f, 1.2330803871154785f, 0.8789023160934448f, -2.017490863800049f, 0.856982946395874f, 0.04652906581759453f, -0.3498983681201935f, -0.8770002722740173f, -0.38342466950416565f, -0.05906866118311882f, 0.7605865001678467f, -0.9748004078865051f, 0.9013839960098267f, -0.43347686529159546f, -1.508413314819336f, 2.2633509635925293f, 0.88945072889328f, 0.7355136275291443f, -0.6224141120910645f, 0.2808963656425476f, 0.1529616415500641f, 0.09397149085998535f, 0.8745917677879333f, -1.226547122001648f, 1.3209213018417358f, 1.0341976881027222f, -0.49460795521736145f, -3.3865373134613037f, 2.0950191020965576f, 0.2728019058704376f, -0.030417079105973244f, 0.08612233400344849f, -1.7298319339752197f, 0.08109258860349655f, 1.2689827680587769f, -0.6729629635810852f, 0.8545035123825073f, -1.1793856620788574f, -0.9540548324584961f, -0.969673752784729f, 0.3141362965106964f, 1.2000207901000977f, -0.1831716001033783f, -0.18045181035995483f, -0.1034746766090393f, -0.10869846493005753f, 0.5782245993614197f, 1.7499538660049438f, 0.002034955658018589f, 0.36500951647758484f, -0.8946718573570251f, -0.219259113073349f, -0.47011202573776245f, -0.9500816464424133f, -0.6808398962020874f, -0.9808987379074097f, -0.7519583702087402f, -1.0324863195419312f, -1.8080289363861084f, 0.21794334053993225f, 0.6857624053955078f, 0.43733805418014526f, -1.9561761617660522f, 1.2915894985198975f, -1.6977379322052002f, -1.2025177478790283f, -1.646543025970459f, -1.3370387554168701f, 1.169712781906128f, -0.5579738020896912f, 0.38680657744407654f, -0.4910755455493927f, -0.32521501183509827f, 0.13093096017837524f, 0.624261200428009f, 0.6792762279510498f, 0.3959449231624603f, 0.66949063539505f, -0.19622361660003662f, -0.32004305720329285f, 0.20433473587036133f, -1.9621531963348389f, -0.5470468401908875f, 1.5721423625946045f, 1.0094727277755737f, 0.4029425382614136f, 1.6145533323287964f, 0.4475519359111786f, 0.5499113202095032f, 0.21999366581439972f, -0.019641423597931862f, 0.20199595391750336f, -0.305207222700119f, -0.11879883706569672f, 0.8017854690551758f, -0.3870505392551422f, 1.560797929763794f, 0.04964430257678032f, -0.33324524760246277f, 0.7588040828704834f, -1.5863133668899536f, -0.6730678081512451f, -1.2264569997787476f, 1.3401838541030884f, -1.162084698677063f, 1.4780147075653076f, 1.5603762865066528f, -0.3231433033943176f, 2.119018793106079f, 0.7877871990203857f, -1.9625552892684937f, -0.526032567024231f, 2.639968156814575f, 1.171855092048645f, 1.2722933292388916f, 0.4042589068412781f, -1.5217982530593872f, -0.5133728981018066f, -1.3806458711624146f, -0.5332193970680237f, 1.0821458101272583f, 0.6407410502433777f, -1.729612946510315f, -2.14782977104187f, 0.4973359704017639f, -1.268904685974121f, -0.24232769012451172f, -1.0745846033096313f, -0.6819858551025391f, 1.0070197582244873f, -0.8279359936714172f, 2.587069511413574f, -0.7386665940284729f, 0.3296529948711395f, 2.8114237785339355f, -0.879487931728363f, -0.33060166239738464f, 0.6457027792930603f, 0.17840361595153809f, -1.1289931535720825f, -0.3602311313152313f, -2.383305549621582f, 1.3506853580474854f, -0.4065081775188446f, -0.30150434374809265f, -1.5348396301269531f, 1.6021355390548706f, 0.6893559694290161f, 0.033526014536619186f, -0.2977294325828552f, -1.1700760126113892f, 1.5237687826156616f, 1.2984881401062012f, 0.6588833928108215f, -0.8222662210464478f, -0.14632517099380493f, 0.7788861393928528f, 0.6403340101242065f, 0.8075235486030579f, 0.5731524229049683f, 1.3903805017471313f, -0.23513318598270416f, -0.5386273264884949f, -0.8155969977378845f, -0.1509992778301239f, -0.5020706653594971f, 0.8036778569221497f, -1.1333380937576294f, -0.7435185313224792f, -0.7013434171676636f, -0.42178988456726074f, -1.5332636833190918f, -1.9342981576919556f, 0.3510490953922272f, -0.6404718160629272f, 0.019135747104883194f, -0.7149955034255981f, -0.7025302648544312f, 0.7700265645980835f, -0.7571792006492615f, -1.0642281770706177f, 1.9611531496047974f, -0.4390997588634491f, 0.8920031785964966f, 1.8720558881759644f, 0.24501417577266693f, 0.7017510533332825f, 0.9099079370498657f, -0.7323962450027466f, -0.6663155555725098f, -0.3277064561843872f, -1.439155101776123f, -1.2797472476959229f, -0.19622953236103058f, 0.19903564453125f, -0.48808231949806213f, -0.6568363904953003f, -0.4720822274684906f, 0.35971710085868835f, 0.5307507514953613f, 1.0079243183135986f, 1.6095112562179565f, 1.6016820669174194f, 0.020138248801231384f, 0.573236346244812f, 0.37203314900398254f, 0.22609540820121765f, -0.754463255405426f, -0.3379971385002136f, 0.41659975051879883f, 0.22890278697013855f, -0.5443984270095825f, 2.050001859664917f, 0.06459449976682663f, 1.448185920715332f, -0.20686893165111542f, 1.5211719274520874f, -1.2341058254241943f, -0.7871248126029968f, -0.7268313765525818f, -0.26047298312187195f, -0.1815493106842041f, 0.40858983993530273f, 0.21912771463394165f, -0.45708730816841125f, -0.651996910572052f, -0.32107922434806824f, 0.20424777269363403f, -0.8860291838645935f, 1.0496968030929565f, 0.5568186640739441f, 0.13751773536205292f, -0.15331187844276428f, 1.2169883251190186f, 1.7785981893539429f, -0.709164023399353f, -0.9840890765190125f, -0.17089858651161194f, 0.2695762813091278f, 0.3193606436252594f, -0.6905565857887268f, 0.3500501811504364f, -0.32124459743499756f, -0.25051894783973694f, 2.1708250045776367f, -1.229917049407959f, 0.3085583448410034f, -0.4612124264240265f, -1.0031081438064575f, -3.3694987297058105f, 0.7832650542259216f, -0.34354400634765625f, 0.34807446599006653f, -1.7184555530548096f, -0.8929248452186584f, -0.7456709742546082f, 0.1508672833442688f, -0.7863383889198303f, -0.350765198469162f, -0.04678618907928467f, -0.6908267736434937f, -0.38857290148735046f, 0.04591083899140358f, -0.3794901967048645f, 1.5770217180252075f, -0.042833227664232254f, 1.353956699371338f, -1.124684453010559f, -0.17863909900188446f, 0.20412735641002655f, 0.9273927807807922f, -0.32438376545906067f, 0.6851024031639099f, 1.0722298622131348f, -1.000272512435913f, -0.8011578917503357f, -1.122711181640625f, -0.7824243903160095f, -0.5122944712638855f, -0.8242338299751282f, -1.0234198570251465f, -1.3928337097167969f, -0.4097520112991333f, 0.8509801030158997f, 0.8881950378417969f, 0.5827952027320862f, 0.785405158996582f, 1.5136889219284058f, 1.251064419746399f, -1.9372137784957886f, -1.2051048278808594f, 1.1982215642929077f, -1.376950740814209f, 1.0670830011367798f, 0.507300615310669f, 0.4764442443847656f, 0.9100666046142578f, -1.501094937324524f, -0.11335238814353943f, -1.2001420259475708f, -0.7933986186981201f, -2.7309763431549072f, -0.014749551191926003f, -0.04001638665795326f, -0.11494863778352737f, 1.458335041999817f, 0.4836253225803375f, 1.7024855613708496f, -0.9683796763420105f, 1.086437702178955f, 1.8661012649536133f, 1.4962280988693237f, 1.022226333618164f, 1.5485637187957764f, -0.08855976164340973f, -0.88787841796875f, -0.6835475564002991f, -0.015076662413775921f, -0.030384592711925507f, -0.8306879997253418f, -1.7835168838500977f, -1.239241361618042f, 0.9374039173126221f, -0.12884187698364258f, 0.9388713836669922f, 0.2919924259185791f, 1.7478625774383545f, -0.8145643472671509f, 0.9247024655342102f, -1.5529402494430542f, 0.13943159580230713f, 1.0945836305618286f, 1.0349615812301636f, 0.03250877186655998f, 0.9967503547668457f, 0.6327362060546875f, 0.7831454873085022f, 0.5098334550857544f, 1.5633010864257812f, -0.46283742785453796f, 0.1497601568698883f, 0.6901775598526001f, -0.38219600915908813f, 0.4878943860530853f, 0.04518410563468933f, 1.9969353675842285f, -0.6955253481864929f, 0.5120656490325928f, -0.9795308709144592f, 2.3521711826324463f, 0.41813287138938904f, -0.1308819055557251f, -0.4394487142562866f, 1.6976182460784912f, 0.813541829586029f, -0.6087417602539062f, 0.9832689762115479f, 2.428079843521118f, 1.1440294981002808f, -0.101145900785923f, 0.09917446225881577f, 0.3814586102962494f, 1.6723263263702393f, -0.4240947961807251f, 1.8199645280838013f, -0.39397287368774414f, -0.4608776867389679f, 0.51866215467453f, 0.7784189581871033f, -1.5908970832824707f, 1.9877948760986328f, 0.02497357502579689f, -0.43002068996429443f, 1.0171878337860107f, -0.548503041267395f, 1.6083859205245972f, 0.5863268971443176f, 0.15758942067623138f, -0.9828112125396729f, 0.18478159606456757f, 1.4150960445404053f, 0.4230518639087677f, 0.5434484481811523f, -1.2116377353668213f, 0.1757367104291916f, 0.25499454140663147f, 0.8070029616355896f, -0.11587464064359665f, 0.45107510685920715f, -0.3600947856903076f, -1.0252388715744019f, 0.37429946660995483f, -0.3791492283344269f, 2.650177001953125f, -1.6031533479690552f, -0.47765374183654785f, -0.5845633149147034f, -0.21724168956279755f, -0.5731971859931946f, 0.37175968289375305f, -0.4610559642314911f, 2.174776315689087f, 0.021914366632699966f, 0.1974239945411682f, -0.7603867650032043f, -0.693874180316925f, -1.0808472633361816f, -0.06485684216022491f, -0.010432514362037182f, 1.2084461450576782f, -0.9010456800460815f, -1.3052830696105957f, -0.12543383240699768f, 1.2808754444122314f, 0.6577393412590027f, 0.46201997995376587f, -0.5072460770606995f, -0.9180205464363098f, 0.0884491354227066f, -1.0534428358078003f, 0.2707360088825226f, -0.25277644395828247f, 0.016658902168273926f, -0.2014150619506836f, 0.05029870569705963f, 1.4853851795196533f, 0.8810616731643677f, 0.9759232997894287f, 0.807918131351471f, -1.3484669923782349f, 1.1456364393234253f, 0.31596988439559937f, -0.4552096426486969f, -0.18135374784469604f, -0.3125359117984772f, 2.018014669418335f, -0.44383805990219116f, -0.7552081942558289f, -1.7850168943405151f, -0.07548937201499939f, -0.5207358598709106f, -1.048349142074585f, 0.8424208760261536f, -0.9976255297660828f, 0.7335740327835083f, 1.2276335954666138f, -0.7577790021896362f, 1.622873306274414f, 1.292340874671936f, -0.26265472173690796f, -0.8315040469169617f, 1.0420149564743042f, 1.154740810394287f, 0.31027355790138245f, -0.6142987608909607f, 1.667606234550476f, 0.014784318394958973f, -1.3251757621765137f, -0.8346104025840759f, -0.2093493789434433f, 0.2113834023475647f, -0.15933676064014435f, 1.438138484954834f, -0.8965448141098022f, 0.4655975103378296f, -0.7802034020423889f, -0.7512221336364746f, -0.20733071863651276f, 0.41877850890159607f, 1.7955039739608765f, -0.6994057297706604f, -0.4367755055427551f, 0.34429505467414856f, -0.5315167307853699f, -1.7878962755203247f, 0.5759621858596802f, -0.37107211351394653f, -0.02148612029850483f, -0.3246121108531952f, 0.7541995644569397f, -1.7302361726760864f, -0.6404775977134705f, -0.024382900446653366f, -0.7490391731262207f, -2.1543679237365723f, -0.15692853927612305f, 0.09731115400791168f, 0.2538682818412781f, 1.5049229860305786f, 1.313053011894226f, -0.7395356297492981f, -1.813647985458374f, -0.2881675958633423f, -0.4023030996322632f, 0.7287482619285583f, -0.49943476915359497f, 0.7028453350067139f, 0.34216001629829407f, -1.3413481712341309f, 0.41704463958740234f, -0.8712550401687622f, -0.16223306953907013f, -0.9481205940246582f, -0.6534963846206665f, -0.3585118353366852f, -0.817564845085144f, 0.3736099898815155f, 0.9122175574302673f, 0.6781302690505981f, 0.055852312594652176f, -1.3394618034362793f, -0.30016884207725525f, 0.15999306738376617f, -0.6769115328788757f, 0.7504494786262512f, 0.216787651181221f, 0.23502039909362793f, 1.8141969442367554f, -1.5329066514968872f, -0.3652367889881134f, -1.5126714706420898f, 0.7930607199668884f, -1.0631688833236694f, 0.9984860420227051f, 1.037604570388794f, -0.04009038954973221f, -1.5908479690551758f, 0.30711954832077026f, 1.1175891160964966f, -1.345697283744812f, 1.0572247505187988f, 1.3808448314666748f, -0.366502046585083f, -0.08894097805023193f, -0.662988007068634f, -0.291555255651474f, 0.10108345001935959f, 1.0123193264007568f, -0.815822958946228f, -0.18586060404777527f, -0.7641056776046753f, -0.3749435842037201f, -0.8628830313682556f, -0.8098255395889282f, -0.21431559324264526f, -1.1151552200317383f, 0.14144384860992432f, -0.17559614777565002f, -0.7265163660049438f, -0.5217191576957703f, 0.37568187713623047f, 0.5935606956481934f, 0.20674307644367218f, 1.4827877283096313f, 0.9587778449058533f, -0.3417870104312897f, -1.0058722496032715f, -1.0774322748184204f, -2.2288825511932373f, 0.8113047480583191f, 0.13623955845832825f, -0.30091696977615356f, -0.2654571533203125f, -1.8403681516647339f, 0.1881687343120575f, -1.4522165060043335f, 1.4667081832885742f, 0.9349684119224548f, 0.25753054022789f, 0.9158172607421875f, -0.5319824814796448f, 0.6224501132965088f, 0.12310407310724258f, -0.17781615257263184f, 2.9330546855926514f, 0.4627670347690582f, -0.6264359354972839f, -1.3771589994430542f, -1.6635388135910034f, -0.2668311595916748f, -1.2189315557479858f, 0.14258776605129242f, -0.31570670008659363f, 1.423683524131775f, 1.0125263929367065f, -0.7147458791732788f, 0.3446122407913208f, 0.8802022337913513f, -0.7164220213890076f, -1.3230516910552979f, 1.0481867790222168f, -1.7053356170654297f, 0.600459098815918f, 1.8295810222625732f, -0.340763121843338f, 0.34682390093803406f, -0.1821630895137787f, -0.12917080521583557f, 0.7775542140007019f, 0.3030397295951843f, -0.8394461274147034f, 0.4900873601436615f, 0.9002130031585693f, -1.4860044717788696f, 0.9374179840087891f, -0.5453174114227295f, 0.485347181558609f, 0.6994611024856567f, -0.28342336416244507f, -0.11592065542936325f, 0.22056539356708527f, -0.8155755400657654f, 1.1422392129898071f, 0.16099725663661957f, 1.1248847246170044f, 0.09503551572561264f, 0.6680463552474976f, 0.22494570910930634f, -0.8516902327537537f, 0.4524131119251251f, -0.7356429100036621f, 1.461809754371643f, 0.056686531752347946f, -0.6240271329879761f, -0.3087240159511566f, 0.18612052500247955f, 0.026730623096227646f, -0.8790956139564514f, 1.3807705640792847f, -0.6990379691123962f, -1.2263665199279785f, -1.9690755605697632f, 2.3512303829193115f, 0.35396817326545715f, -0.10606183111667633f, -2.408507823944092f, -1.2640177011489868f, -0.4252816140651703f, 0.24079595506191254f, -0.24080689251422882f, -0.6830735802650452f, 0.28334593772888184f, -0.7549530863761902f, 0.685922384262085f, -0.28879889845848083f, 0.16147172451019287f, -0.18717071413993835f, 0.7061312794685364f, 0.12637053430080414f, -0.2261367291212082f, -0.40762385725975037f, -0.6097135543823242f, -1.347667932510376f, 0.8310710787773132f, 0.23480524122714996f, 0.23941050469875336f, -0.15233780443668365f, 1.35030198097229f, 0.3163374066352844f, -0.686985969543457f, 0.31536293029785156f, -1.2173928022384644f, -0.23235148191452026f, -0.11819366365671158f, -0.7065277099609375f, 0.8570377826690674f, -0.6065717935562134f, -2.5028295516967773f, -0.5181497931480408f, -0.3955882489681244f, -0.007207863032817841f, -1.8654048442840576f, -1.2592604160308838f, 0.44495517015457153f, 1.5878770351409912f, -1.009257197380066f, 2.5517168045043945f, 0.27151748538017273f, -0.45277684926986694f, 0.7783017754554749f, 0.5624421238899231f, -1.7339200973510742f, 0.1151348203420639f, 0.5969645380973816f, 1.4009592533111572f, 1.26650869846344f, 0.5665708780288696f, 1.4182316064834595f, 0.8785027265548706f, 1.713185429573059f, 0.5062318444252014f, 0.9417456984519958f, 1.1770522594451904f, -0.8855880498886108f, -0.897435188293457f, -0.4500139355659485f, -1.2818336486816406f, -0.09055130183696747f, 1.568543553352356f, 0.6288840770721436f, -0.13983508944511414f, -0.8341673016548157f, -1.0642942190170288f, -0.4966285824775696f, -0.8965734243392944f, 0.2338687777519226f, 0.13322870433330536f, -1.087693452835083f, -0.6877674460411072f, -0.034093379974365234f, -1.2961610555648804f, 1.018326997756958f, 2.094313621520996f, 0.22312399744987488f, 0.04033441096544266f, -0.11042066663503647f, 1.209588646888733f, 0.8461918830871582f, 1.0998334884643555f, -2.8670525550842285f, 0.43270260095596313f, 2.3624584674835205f, -0.9460180401802063f, 0.40504685044288635f, 0.21921862661838531f, -1.154799461364746f, 1.519834041595459f, -1.8380980491638184f, 1.218536376953125f, -0.1873958259820938f, 1.0788236856460571f, 0.14058129489421844f, -0.9915911555290222f, -1.5487273931503296f, -1.3351367712020874f, -1.668448805809021f, -1.84798002243042f, 0.02599930204451084f, 0.7676464319229126f, -1.9874215126037598f, 0.9982309937477112f, 0.003224122803658247f, 0.07818755507469177f, -1.0705227851867676f, -1.0210168361663818f, 1.7321865558624268f, 0.5420076847076416f, 0.3053724765777588f, 1.1364459991455078f, 0.6844477653503418f, 0.2691081464290619f, 0.8210340738296509f, 0.27511492371559143f, -0.38933637738227844f, -0.20226091146469116f, -0.19409379363059998f, -0.07234596461057663f, -0.7745715975761414f, 0.2601335644721985f, 1.2001798152923584f, -1.078745722770691f, 1.2962092161178589f, -0.18015773594379425f, -1.9301296472549438f, 0.28462305665016174f, 0.7962566614151001f, -0.26019519567489624f, 0.30697566270828247f, -0.7991265654563904f, 0.6080788373947144f, -0.8028495907783508f, -0.46709370613098145f, 0.5347719192504883f, 1.8560413122177124f, -0.11687781661748886f, 0.6693155169487f, -0.06095214933156967f, -0.1428632140159607f, -1.3090907335281372f, 0.03258459270000458f, 1.4419121742248535f, -0.6330651044845581f, -0.03063541278243065f, 1.5843851566314697f, -0.467591792345047f, -0.03230065852403641f, 0.7200000286102295f, -0.988913357257843f, -1.0607333183288574f, 0.7062268257141113f, -0.2287754863500595f, -0.030369358137249947f, 0.7900131344795227f, -0.22298280894756317f, 1.508110761642456f, 0.599722146987915f, -0.33899757266044617f, 0.1548950970172882f, -0.2794382572174072f, 0.8708680272102356f, 0.7626234889030457f, 0.6961768865585327f, 0.44085901975631714f, -0.4885202944278717f, 0.7180350422859192f, 1.8486416339874268f, -2.431525230407715f, -0.4701055586338043f, 0.4380730092525482f, 0.7702670693397522f, -0.14850954711437225f, 0.24941392242908478f, 0.539443850517273f, 0.058982912451028824f, 0.28753769397735596f, -0.6253249645233154f, 1.5168472528457642f, -0.610085129737854f, -1.8375158309936523f, 1.8140860795974731f, 0.9514070153236389f, -1.2033350467681885f, -0.5142884254455566f, -0.8533352017402649f, -0.5986783504486084f, -0.2986593544483185f, -0.7190134525299072f, -1.4356969594955444f, 0.6340908408164978f, -0.5407924652099609f, 1.3264530897140503f, 0.29634353518486023f, 0.17156629264354706f, 0.8127211332321167f, -0.636906087398529f, -1.1077786684036255f, 1.5005162954330444f, 0.5292592644691467f, -1.5582870244979858f, -0.22035345435142517f, -0.8878109455108643f, -0.23165416717529297f, -0.48117324709892273f, -0.8784040212631226f, 0.039074741303920746f, 0.03500647097826004f, -1.3603416681289673f, -0.3714776337146759f, -0.040006835013628006f, 0.5846149921417236f, 0.5806360840797424f, 0.7450182437896729f, 1.4870532751083374f, -0.051454752683639526f, 0.2527291476726532f, 0.7950483560562134f, 1.2641831636428833f, -0.0403965562582016f, -0.19124414026737213f, -0.4378063976764679f, -0.009368259459733963f, 0.25679612159729004f, -0.003075298387557268f, -1.1646342277526855f, -0.46010419726371765f, 0.18153296411037445f, 0.18499712646007538f, -0.23162956535816193f, 2.05483078956604f, 0.4368106722831726f, 1.1093194484710693f, 1.684390902519226f, -0.11415659636259079f, 0.37000924348831177f, 0.2512214779853821f, 1.1637510061264038f, -0.876596987247467f, -0.6737048625946045f, -0.652714192867279f, -0.753913164138794f, -0.10021805018186569f, -2.3608896732330322f, 2.2464849948883057f, -1.1070212125778198f, 1.026766061782837f, 0.2865336239337921f, -0.10950178653001785f, 2.380542039871216f, -0.41917189955711365f, 0.6831162571907043f, 1.7866971492767334f, -0.022699790075421333f, 0.7491136193275452f, -1.1757766008377075f, 0.2672518789768219f, -0.22947388887405396f, 0.5409282445907593f, 0.6939815282821655f, 1.8563321828842163f}; diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/data/layer.h b/sw/vme/apps/tew32/fp32-matmul-vle-vse/data/layer.h deleted file mode 100644 index 54e4009f2..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/data/layer.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2024 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include - -/** - * @brief VME GEMM layer configuration. - * - * Describes a C = A * B problem with: - * A : M × K (stored row-major; kernel receives pre-transposed At : K × M) - * B : K × N (stored row-major) - * C : M × N (fp32 accumulator, tew=32) - * - * TM / TN are the micro-tile sizes used by the 2×2 blocked VME kernel. - * Typical values: TM = TN = 16. - */ -typedef struct { - uint32_t M; /**< Rows of A / rows of C */ - uint32_t N; /**< Columns of B / columns of C */ - uint32_t K; /**< Inner (reduction) dimension */ - uint32_t TM; /**< Micro-tile height (M direction) */ - uint32_t TN; /**< Micro-tile width (N direction) */ -} vme_gemm_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/kernel/gemm.c b/sw/vme/apps/tew32/fp32-matmul-vle-vse/kernel/gemm.c deleted file mode 100644 index 232e30d57..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/kernel/gemm.c +++ /dev/null @@ -1,318 +0,0 @@ - -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include "gemm.h" - -enum { - GEMM_TSS_MT0_ROW0 = 0u << 27, - GEMM_TSS_MT4_ROW0 = 4u << 27, - GEMM_TSS_MT8_ROW0 = 8u << 27, - GEMM_TSS_MT12_ROW0 = 12u << 27, -}; - -__attribute__((noinline, aligned(64))) void gemm_fp32( - float *C, const float *Apack, const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi) -{ - const int ti = (int)ti_lo; - const int row_blocks = (int)gemm_l.M / 64; - const int col_blocks = (int)gemm_l.N / 32; - const int num_blocks = row_blocks * col_blocks; - const int col_tiles = (int)gemm_l.N / 16; // TILE_DIM - const int tile_stride = 16 * (int)gemm_l.K; - - // The M64 two-core partition assigns one row-tile pair to each core. - (void)ti_hi; - - if (ti_lo != 0) - asm volatile("nop" ::: "memory"); - - uintptr_t a0p = (uintptr_t)(Apack + ti * tile_stride); - uintptr_t a1p = (uintptr_t)(Apack + (ti + 1) * tile_stride); - uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); - uintptr_t tss0 = GEMM_TSS_MT0_ROW0; - uintptr_t tss4 = GEMM_TSS_MT4_ROW0; - uintptr_t tss8 = GEMM_TSS_MT8_ROW0; - uintptr_t tss12 = GEMM_TSS_MT12_ROW0; - uintptr_t p00 = (uintptr_t)(C + ti * col_tiles * 16 * 16); - uintptr_t p01 = p00 + 16 * 16 * sizeof(float); - uintptr_t p10 = (uintptr_t)(C + (ti + 1) * col_tiles * 16 * 16); - uintptr_t p11 = p10 + 16 * 16 * sizeof(float); - const uintptr_t tile_dim = 16; - const uintptr_t TK = 8; - uintptr_t block_counter = (uintptr_t)num_blocks; - uintptr_t loop_counter; - - asm volatile( - - // Every block enters the same bias-load and first-K-group schedule. - "10:\n" - // K-Group 0 - "msetmtypei 1, 2\n" - "msettn x0, %[tile_dim]\n" - "msettm x0, %[tile_dim]\n" - "msettk x0, %[tk]\n" - "vsetvli %[loop], x0, e32, m4, ta, ma\n" - // Load all 16 rows of mt0 before its first MAC. Each LMUL=4 load - // supplies four physical vector registers, hence four loads per tile. - "vle32.v v24, (%[p00])\n" "addi %[p00], %[p00], 256\n" - "vle32.v v28, (%[p00])\n" "addi %[p00], %[p00], 256\n" - "vtmv.t.v %[tss0], v24\n" "addi %[tss0], %[tss0], 4\n" - "vtmv.t.v %[tss0], v28\n" "addi %[tss0], %[tss0], 4\n" - "vle32.v v8, (%[p00])\n" "addi %[p00], %[p00], 256\n" - "vle32.v v12, (%[p00])\n" "addi %[p00], %[p00], -768\n" - "vtmv.t.v %[tss0], v8\n" "addi %[tss0], %[tss0], 4\n" - "vtmv.t.v %[tss0], v12\n" "addi %[tss0], %[tss0], -12\n" - - // A0 is reused by mt0 and mt4. B0 remains at its current address - // because v16..v23 must be restored after loading the mt8 bias. - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vle32.v v20, (%[b0p])\n" - - // mt0 compute || load and install mt4 bias. - "vtfmm.tvv mt0, v0, v16\n" - "vle32.v v24, (%[p01])\n" "addi %[p01], %[p01], 256\n" - "vle32.v v28, (%[p01])\n" "addi %[p01], %[p01], 256\n" - "vtmv.t.v %[tss4], v24\n" "addi %[tss4], %[tss4], 4\n" - "vtmv.t.v %[tss4], v28\n" "addi %[tss4], %[tss4], 4\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v8, (%[p01])\n" "addi %[p01], %[p01], 256\n" - "vle32.v v12, (%[p01])\n" "addi %[p01], %[p01], -768\n" - "vtmv.t.v %[tss4], v8\n" "addi %[tss4], %[tss4], 4\n" - "vtmv.t.v %[tss4], v12\n" "addi %[tss4], %[tss4], -12\n" - - // Replace the consumed mt4-bias group with B1 exactly once. B1 is - // then reused by mt4 and mt12. - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - "vle32.v v28, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - - // mt4 compute || seed mt12 in v8..v15 and v16..v23. - "vle32.v v8, (%[p11])\n" "addi %[p11], %[p11], 256\n" - "vle32.v v12, (%[p11])\n" "addi %[p11], %[p11], 256\n" - "vtfmm.tvv mt4, v0, v24\n" - "vtmv.t.v %[tss12], v8\n" "addi %[tss12], %[tss12], 4\n" - "vle32.v v16, (%[p11])\n" "addi %[p11], %[p11], 256\n" - "vtmv.t.v %[tss12], v12\n" "addi %[tss12], %[tss12], 4\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtmv.t.v %[tss12], v16\n" "addi %[tss12], %[tss12], 4\n" - "vle32.v v20, (%[p11])\n" "addi %[p11], %[p11], -768\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - "vle32.v v12, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - "vtmv.t.v %[tss12], v20\n" "addi %[tss12], %[tss12], -12\n" - - // mt12 compute || seed mt8 in v16..v23 and v0..v7. - "vle32.v v16, (%[p10])\n" "addi %[p10], %[p10], 256\n" - "vle32.v v20, (%[p10])\n" "addi %[p10], %[p10], 256\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtmv.t.v %[tss8], v16\n" "addi %[tss8], %[tss8], 4\n" - "vle32.v v0, (%[p10])\n" "addi %[p10], %[p10], 256\n" - "vtmv.t.v %[tss8], v20\n" "addi %[tss8], %[tss8], 4\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtmv.t.v %[tss8], v0\n" "addi %[tss8], %[tss8], 4\n" - "vle32.v v4, (%[p10])\n" "addi %[p10], %[p10], -768\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vle32.v v20, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vtmv.t.v %[tss8], v4\n" "addi %[tss8], %[tss8], -12\n" - - // mt8 compute. A0 may be preloaded because mt8 reads A1 from v8..v15. - // Keep B0 intact until the final v15/v23 consumer has issued. - "vtfmm.tvv mt8, v8, v16\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - "vle32.v v4, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - "vtfmm.tvv mt8, v12, v20\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vle32.v v20, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - - // K-Group 1 to N-3 - // vle B1 - // vtfmm mt0, A0, B0 - "9:\n" - "li %[loop], 5\n" - "1:\n" - "vtfmm.tvv mt0, v0, v16\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v28, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - - // vle A1 - // vtfmm mt4, A0, B1 - "vtfmm.tvv mt4, v0, v24\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - "vtfmm.tvv mt4, v4, v28\n" - "vle32.v v12, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - - // preload vle A0 - // vtfmm mt8, A1, B0 - "vtfmm.tvv mt8, v8, v16\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - "vtfmm.tvv mt8, v12, v20\n" - "vle32.v v4, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - - // preload vle B0 - // vtfmm mt12, A1, B1 - "addi %[loop], %[loop], -1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v20, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group K-2 & K-Group K-1 - // vle B0' - // vle A0' - // vtfmm mt0, A0, B0 - "vtfmm.tvv mt0, v0, v16\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vle32.v v28, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - "vle32.v v12, (%[a0p])\n" "addi %[a0p], %[a0p], 256\n" - - // vle B1 - // vtfmm mt0, A0', B0' - "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v20, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - - // vle B1' - // vtfmm mt4, A0, B1 - "vtfmm.tvv mt4, v0, v16\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - "vtfmm.tvv mt4, v4, v20\n" - "vle32.v v28, (%[b1p])\n" "addi %[b1p], %[b1p], 256\n" - - // vle A1 - // vtfmm mt4, A0', B1' || vtmv mt0 (row0 - row7) + vse - "vtmv.v.t v0, %[tss0]\n" "addi %[tss0], %[tss0], 4\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtmv.v.t v4, %[tss0]\n" "addi %[tss0], %[tss0], 4\n" - - "vse32.v v0, (%[p00])\n" "addi %[p00], %[p00], 256\n" - "vse32.v v4, (%[p00])\n" "addi %[p00], %[p00], 256\n" - "vtfmm.tvv mt4, v12, v28\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - "vle32.v v4, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - - // vle A1' - // vtfmm mt12, A1, B1 || vtmv mt0 (row8 - row15) + vse - "vtmv.v.t v8, %[tss0]\n" "addi %[tss0], %[tss0], 4\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtmv.v.t v12, %[tss0]\n" "addi %[tss0], %[tss0], 4\n" - - "vse32.v v8, (%[p00])\n" "addi %[p00], %[p00], 256\n" - "vse32.v v12, (%[p00])\n" "addi %[p00], %[p00], 1280\n" - "vtfmm.tvv mt12, v4, v20\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - "vle32.v v12, (%[a1p])\n" "addi %[a1p], %[a1p], 256\n" - - // vle reload B0 (restore B0 pointer) - // vtfmm mt12, A1', B1' || vtmv mt4 (row0 - row7) + vse - "vtmv.v.t v16, %[tss4]\n" "addi %[tss4], %[tss4], 4\n" - "vse32.v v16, (%[p01])\n" "addi %[p01], %[p01], 256\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtmv.v.t v20, %[tss4]\n" "addi %[tss4], %[tss4], 4\n" - "vse32.v v20, (%[p01])\n" "addi %[p01], %[p01], 256\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vle32.v v20, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - - // vle reload B0' - // vtfmm mt8, A1, B0 || vtmv mt4 (row8 - row15) + vse - "vtmv.v.t v24, %[tss4]\n" "addi %[tss4], %[tss4], 4\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtmv.v.t v28, %[tss4]\n" "addi %[tss4], %[tss4], 4\n" - "vse32.v v24, (%[p01])\n" "addi %[p01], %[p01], 256\n" - "vse32.v v28, (%[p01])\n" "addi %[p01], %[p01], 1280\n" - "vtfmm.tvv mt8, v4, v20\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - "vle32.v v28, (%[b0p])\n" "addi %[b0p], %[b0p], 256\n" - - // vtfmm mt8, A1', B0' || vtmv mt12 (row0 - row7) + vse - // vtmv mt12 (row8 - row15) + vse - "addi %[loop], %[tss12], 8\n" - - "vtmv.v.t v0, %[tss12]\n" "addi %[tss12], %[tss12], 4\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtmv.v.t v4, %[tss12]\n" "addi %[tss12], %[tss12], 4\n" - - "vse32.v v0, (%[p11])\n" "addi %[p11], %[p11], 256\n" - "vse32.v v4, (%[p11])\n" "addi %[p11], %[p11], 256\n" - "vtmv.v.t v16, %[loop]\n" "addi %[loop], %[loop], 4\n" - "vtmv.v.t v20, %[loop]\n" "addi %[loop], %[loop], 4\n" - "vse32.v v16, (%[p11])\n" "addi %[p11], %[p11], 256\n" - "vse32.v v20, (%[p11])\n" "addi %[p11], %[p11], 1280\n" - "mv %[tss12], %[loop]\n" - "addi %[blocks], %[blocks], -1\n" - "j 7f\n" - - // Store mt8 for every block. Non-final blocks then return to the - // shared bias-load prologue above. - "7:\n" - - "vtmv.v.t v0, %[tss8]\n" "addi %[tss8], %[tss8], 4\n" - "vtmv.v.t v4, %[tss8]\n" "addi %[tss8], %[tss8], 4\n" - - "vse32.v v0, (%[p10])\n" "addi %[p10], %[p10], 256\n" - "vse32.v v4, (%[p10])\n" "addi %[p10], %[p10], 256\n" - "vtmv.v.t v8, %[tss8]\n" "addi %[tss8], %[tss8], 4\n" - "vtmv.v.t v12, %[tss8]\n" "addi %[tss8], %[tss8], 4\n" - "vse32.v v8, (%[p10])\n" "addi %[p10], %[p10], 256\n" - "vse32.v v12, (%[p10])\n" "addi %[p10], %[p10], 1280\n" - "beqz %[blocks], 8f\n" - - // Retarget the scalar streams only after the old mt8 store has - // consumed its p10/tss8 state. - "addi %[tss0], %[tss0], -16\n" - "addi %[tss4], %[tss4], -16\n" - "addi %[tss8], %[tss8], -16\n" - "addi %[tss12], %[tss12], -16\n" - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - "j 10b\n" - "8:\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss0] "+r"(tss0), - [tss4] "+r"(tss4), - [tss8] "+r"(tss8), - [tss12] "+r"(tss12), - [p00] "+r"(p00), - [p01] "+r"(p01), - [p10] "+r"(p10), - [p11] "+r"(p11), - [blocks] "+r"(block_counter), - [loop] "=&r"(loop_counter) - : - [tile_dim] "r"(tile_dim), - [tk] "r"(TK) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/kernel/gemm.h b/sw/vme/apps/tew32/fp32-matmul-vle-vse/kernel/gemm.h deleted file mode 100644 index 622b9439d..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/kernel/gemm.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// gemm.h — interface for the fp32×fp32=fp32 VME GEMM kernel. - -#include - -// Read the hardware cycle counter. -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; -} - -// Reading fcsr stalls Snitch while any Spatz scoreboard entry is live. Use it -// to turn an asynchronous kernel return into an architectural completion point. -static inline void wait_spatz(void) -{ - uint32_t fcsr; - asm volatile("csrr %0, fcsr" : "=r"(fcsr) :: "memory"); -} - -// FP32 VME GEMM over TE=16 with architectural tk=1 and RES_K=8 software -// residency. [ti_lo, ti_hi) selects an even-aligned range of 16-row tiles. -void gemm_fp32(float *C, const float *Apack, const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi); diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/main.c b/sw/vme/apps/tew32/fp32-matmul-vle-vse/main.c deleted file mode 100644 index f9d6629df..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/main.c +++ /dev/null @@ -1,197 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// main.c — fp32×fp32=fp32 VME GEMM functional check - -#include -#include -#include - -#include DATAHEADER -#include "kernel/gemm.c" - -// Shared L1 buffers: core 0 allocates and DMAs; the pointers live in shared -// TCDM (.data) so every core sees the same addresses. Declared at file scope -// (not on main's stack) precisely so the assignment by core 0 is visible to the -// other cores after the barrier -- the sp-fmatmul multi-core pattern. -static const float *Apack, *Bpack, *Cref; -static float *C; -static uint32_t allocation_failed; -static volatile uint32_t core_cycles[16]; - -static inline float gemm_bias(uint32_t row, uint32_t col) -{ - const int32_t value = (int32_t)((row * 3u + col * 5u) & 15u) - 8; - return (float)value * 0.03125f; -} - -int main(void) -{ - const uint32_t cid = snrt_cluster_core_idx(); - const uint32_t ncores = snrt_cluster_core_num(); -#ifdef SPATZ_GEMM_SINGLE_CORE - const uint32_t active_cores = 1; -#else - const uint32_t active_cores = ncores; -#endif - - const uint32_t M = gemm_l.M; - const uint32_t N = gemm_l.N; - const uint32_t K = gemm_l.K; - const uint32_t TM = 16; - const uint32_t TN = 16; - const uint32_t output_ld = N; - - // Core 0 owns the DMA engine and publishes all shared L1 pointers. - const uint32_t c_bytes = M * output_ld * sizeof(float); - const uint32_t c_ref_bytes = M * N * sizeof(float); - const uint32_t a_bytes = K * M * sizeof(float); - const uint32_t b_bytes = K * N * sizeof(float); - - if (cid == 0) { - C = (float *)snrt_l1alloc(c_bytes); - Cref = (float *)snrt_l1alloc(c_ref_bytes); - Bpack = (float *)snrt_l1alloc(b_bytes); - Apack = (float *)snrt_l1alloc(a_bytes); - allocation_failed = C == NULL || Cref == NULL || Apack == NULL || Bpack == NULL; - if (!allocation_failed) { - snrt_dma_start_1d((void *)Cref, gemm_C_dram, c_ref_bytes); - snrt_dma_start_1d((void *)Apack, gemm_Apack_dram, a_bytes); - snrt_dma_start_1d((void *)Bpack, gemm_Bpack_dram, b_bytes); - snrt_dma_wait_all(); - for (uint32_t row = 0; row < M; ++row) { - for (uint32_t col = 0; col < N; ++col) { - const uint32_t tile_row = row / TM; - const uint32_t tile_col = col / TN; - const uint32_t row_in_tile = row % TM; - const uint32_t col_in_tile = col % TN; - const uint32_t packed_idx = - (tile_row * (N / TN) + tile_col) * (TM * TN) + - row_in_tile * TN + col_in_tile; - C[packed_idx] = gemm_bias(row, col); - } - } - } - } - - // Partition complete 2x2 tile blocks. This keeps the range pair-aligned and - // leaves excess cores idle when M=32 has only one row-tile pair. - const uint32_t npairs = (M / TM) / 2; - const uint32_t pairs_per_core = - (npairs + active_cores - 1) / active_cores; - uint32_t pair_lo = cid < active_cores ? cid * pairs_per_core : npairs; - uint32_t pair_hi = cid < active_cores ? pair_lo + pairs_per_core : npairs; - if (pair_hi > npairs) pair_hi = npairs; - if (pair_lo > npairs) pair_lo = npairs; - const uint32_t ti_lo = 2 * pair_lo; - const uint32_t ti_hi = 2 * pair_hi; - - // Barrier: all cores wait until the buffers are DMA'd in and pointers set. - snrt_cluster_hw_barrier(); - - if (allocation_failed) { - if (cid == 0) printf("GEMM allocation failed\n"); - return 2; - } - - if (K % 8 != 0) - return 2; - - // Synchronize the launch, but keep both barriers outside the measured - // interval. Each core measures only its own GEMM slice. - snrt_cluster_hw_barrier(); - if (cid < active_cores) { - uint32_t t0 = get_cycle(); - gemm_fp32(C, Apack, Bpack, ti_lo, ti_hi); - uint32_t t1 = get_cycle(); - core_cycles[cid] = t1 - t0; - } else { - core_cycles[cid] = 0; - } - - // Publish all per-core durations before core 0 selects the parallel - // wall-clock time, defined as the slowest core's kernel duration. - snrt_cluster_hw_barrier(); - - // Only core 0 verifies and reports. - if (cid != 0) - return 0; - - uint32_t cycles = 0; - for (uint32_t i = 0; i < active_cores; ++i) { - if (core_cycles[i] > cycles) - cycles = core_cycles[i]; - } - - enum { CE = 8 }; - const uint64_t useful_fmas = (uint64_t)M * N * K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE * active_cores; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - // Element-wise error check. - // fdiv.s is not supported by the Spatz FPU; use fmul.s to avoid it. - // Instead of rel_err = abs_err / den > tol, check abs_err > tol * den. - int n_err = 0; - float max_abs = 0.0f; - const float tol = 1.0e-3f; - - uint32_t row_errs[M]; - memset(row_errs, 0, sizeof(row_errs)); - for (uint32_t idx = 0; idx < M * N; idx++) { - const uint32_t row = idx / N; - const uint32_t col = idx % N; - const uint32_t tile_row = row / TM; - const uint32_t tile_col = col / TN; - const uint32_t row_in_tile = row % TM; - const uint32_t col_in_tile = col % TN; - const uint32_t packed_idx = - (tile_row * (N / TN) + tile_col) * (TM * TN) + - row_in_tile * TN + col_in_tile; - const float ref = Cref[idx] + gemm_bias(row, col); - float diff = C[packed_idx] - ref; - float abs_err = diff < 0.0f ? -diff : diff; - float den = ref < 0.0f ? -ref : ref; - if (den < 1.0e-9f) den = 1.0e-9f; - if (abs_err > max_abs) max_abs = abs_err; - if (abs_err > tol && abs_err > tol * den) { - row_errs[idx / N]++; - n_err++; - } - } - for (uint32_t r = 0; r < M; r++) { - if (row_errs[r]) - printf("row %u: %u errors\n", (unsigned)r, (unsigned)row_errs[r]); - } - - printf("\n=== VME fp32 GEMM %ux%ux%u (TM=%u TN=%u, %u core%s) ===\n", - (unsigned)M, (unsigned)N, (unsigned)K, - (unsigned)TM, (unsigned)TN, - (unsigned)active_cores, active_cores == 1 ? "" : "s"); - printf("Cycles: %u\n", (unsigned)cycles); - printf("Utilization: %u.%02u%%\n", (unsigned)(utilization_bp / 100), - (unsigned)(utilization_bp % 100)); - printf("MaxAbsErr: 0x%08x\n", *(unsigned *)&max_abs); - printf("Errors: %d\n", n_err); - printf("Status: %s\n", n_err == 0 ? "PASS" : "FAIL"); - - // No trailing barrier: the non-zero cores have already returned, so an extra - // barrier here would have no partner and deadlock. - return n_err == 0 ? 0 : 1; -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vse/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vle-vse/script/gen_data.py deleted file mode 100644 index 637655fc8..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vse/script/gen_data.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_gemm_layer gemm_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - f" .TM = {TM},", - f" .TN = {TN},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float gemm_C_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = gemm_Apack_dram + (mt*(K/4)+g)*64.", - f'static float gemm_Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = gemm_Bpack_dram + (nt*(K/4)+g)*64.", - f'static float gemm_Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/data/data_64_64_64.h b/sw/vme/apps/tew32/fp32-matmul-vle-vtse/data/data_64_64_64.h deleted file mode 100644 index 49d1b3900..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/data/data_64_64_64.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Auto-generated by script/gen_data.py — do not edit by hand. - -#include "layer.h" - -const vme_gemm_layer gemm_l = { - .M = 64, - .N = 64, - .K = 64, - .TM = 8, - .TN = 8, -}; - -// C[M*N] row-major fp32 golden reference, C = A*B -static float gemm_C_dram[64*64] __attribute__((section(".data"))) = -{11.479634284973145f, -6.046323299407959f, -4.815614700317383f, -2.040816068649292f, -8.940674781799316f, 1.9331291913986206f, 4.764830112457275f, 14.248414993286133f, -3.9397127628326416f, -2.1485483646392822f, -0.8390139937400818f, -1.3708610534667969f, 13.092900276184082f, 3.2064008712768555f, 5.635993480682373f, 0.7455204725265503f, 7.919027805328369f, -11.349638938903809f, -4.450965404510498f, 15.0212984085083f, 0.6696339845657349f, -0.1861145943403244f, 4.869348049163818f, 3.4079155921936035f, -3.9942312240600586f, 8.851547241210938f, 8.788155555725098f, 6.571914127562195e-05f, 9.69917106628418f, -8.461928367614746f, -8.80200481414795f, 4.7893218994140625f, -0.18487417697906494f, -11.189379692077637f, -12.456897735595703f, -3.528578758239746f, 6.636075019836426f, 4.215647220611572f, -2.5566506385803223f, -2.2709872722625732f, 24.085134506225586f, -3.519670009613037f, -1.5794446468353271f, 13.206070899963379f, -9.0339937210083f, -0.49932846426963806f, 8.33430290222168f, -3.4920732975006104f, 24.13520622253418f, -2.866286516189575f, -5.177119255065918f, 8.296358108520508f, -2.486847162246704f, 7.566592693328857f, -18.43720817565918f, 17.894702911376953f, -17.132667541503906f, 4.636127471923828f, -15.027854919433594f, 14.163813591003418f, -16.567644119262695f, 4.0814080238342285f, -19.760257720947266f, 0.5436081290245056f, -1.0178847312927246f, -0.959050178527832f, -7.164222717285156f, 0.9985886216163635f, -8.26685619354248f, -2.2400271892547607f, 0.6488896608352661f, -3.7606098651885986f, 6.696377277374268f, 1.2368918657302856f, 0.3883592486381531f, -2.765044689178467f, 1.5367969274520874f, 5.624364852905273f, -1.1590087413787842f, 5.226413249969482f, 8.801167488098145f, 0.28941860795021057f, 7.781001091003418f, 1.1585612297058105f, -15.459006309509277f, -3.4766643047332764f, -8.453717231750488f, 10.173603057861328f, 3.6514530181884766f, 1.6193315982818604f, 5.127340793609619f, -0.9438349008560181f, -3.7507288455963135f, 13.667619705200195f, -10.900826454162598f, 11.15420913696289f, -2.949625253677368f, -2.746349811553955f, -16.438350677490234f, -17.064266204833984f, -4.720984935760498f, 4.893701553344727f, -2.473956823348999f, 4.502293586730957f, -5.264101982116699f, -8.454588890075684f, -5.186229705810547f, 5.584354877471924f, -2.95721435546875f, -3.410273790359497f, -6.588162899017334f, -0.8277148604393005f, -2.7242908477783203f, 4.025758266448975f, -1.4152922630310059f, 2.386408805847168f, 7.04514741897583f, -0.22987337410449982f, -0.45179229974746704f, 12.081316947937012f, -5.206918716430664f, 5.094143390655518f, -5.665627956390381f, 10.579627990722656f, -6.4196953773498535f, -8.07258415222168f, -19.385873794555664f, -8.815452575683594f, 1.0405519008636475f, -15.006485939025879f, -7.755399227142334f, 10.724888801574707f, 6.764276504516602f, 8.178961753845215f, 7.572741985321045f, 18.397531509399414f, -1.7755249738693237f, -1.9764132499694824f, 2.622986078262329f, -3.577617645263672f, -0.9461591839790344f, 14.711101531982422f, 8.183825492858887f, -7.469543933868408f, 14.31879711151123f, -6.517776012420654f, 2.1930079460144043f, 6.8929572105407715f, -10.022858619689941f, -1.7278424501419067f, 12.308111190795898f, 13.900723457336426f, 7.140420913696289f, 12.356964111328125f, 4.42258358001709f, -0.5004820227622986f, -1.5587430000305176f, 15.559115409851074f, 0.4262810945510864f, -2.8206470012664795f, 15.083824157714844f, -4.9072794914245605f, 2.5857160091400146f, 4.584408283233643f, -17.988496780395508f, -11.013274192810059f, 2.8712387084960938f, 4.990177631378174f, 0.6336867213249207f, 8.750150680541992f, 18.924428939819336f, 3.4662630558013916f, -14.959811210632324f, 3.8478240966796875f, 17.260263442993164f, -13.368204116821289f, 2.653244972229004f, -4.103200435638428f, -12.37991714477539f, -0.2966856062412262f, 5.892507076263428f, 6.881699562072754f, -3.230393648147583f, -10.482145309448242f, -13.298931121826172f, -6.5993547439575195f, 0.7129412293434143f, -2.0269880294799805f, 4.382270336151123f, -6.041990756988525f, -17.212663650512695f, -6.112694263458252f, 13.97390365600586f, -4.114153861999512f, -0.9799354672431946f, -4.399726390838623f, -5.032261848449707f, 0.5352955460548401f, -1.2309777736663818f, 10.451981544494629f, -5.048009872436523f, 3.677152633666992f, 11.969364166259766f, -0.417408287525177f, -3.176297426223755f, 5.826728343963623f, -5.306847095489502f, 10.880280494689941f, 12.627052307128906f, -5.021015644073486f, 0.382880836725235f, -12.032983779907227f, -2.4724607467651367f, -2.954784393310547f, 4.853247165679932f, -2.217081308364868f, -7.221776485443115f, 5.463408470153809f, -0.44735994935035706f, -10.677308082580566f, 1.9355524778366089f, -2.376770496368408f, 1.6620814800262451f, -12.09306526184082f, 9.559584617614746f, -1.3697234392166138f, -8.889398574829102f, 13.613365173339844f, -6.308327674865723f, -2.4649970531463623f, 2.388371467590332f, 4.159750938415527f, 0.5139855742454529f, 4.015327453613281f, -3.1259841918945312f, -1.0106412172317505f, 3.4432032108306885f, 4.1760029792785645f, -6.856860637664795f, -2.4840879440307617f, 0.686843752861023f, -1.0297985076904297f, 2.4013655185699463f, -3.5610408782958984f, 0.8839467167854309f, -2.944474458694458f, 12.640097618103027f, -3.0358707904815674f, -8.915057182312012f, 20.95890998840332f, 11.799171447753906f, 0.5034058690071106f, -1.9294973611831665f, -1.5430312156677246f, -1.1323392391204834f, 10.141955375671387f, -0.7313078045845032f, 3.43355655670166f, 12.49581241607666f, -0.004268331453204155f, 10.47712230682373f, 19.086078643798828f, -5.440404891967773f, -6.789121150970459f, 0.7849673628807068f, -0.7528467178344727f, -3.794801712036133f, -7.670317649841309f, 14.326471328735352f, -9.476102828979492f, 6.799417972564697f, -10.577319145202637f, -12.32802963256836f, 3.4096364974975586f, -24.734689712524414f, 3.53064227104187f, -1.0564911365509033f, 1.1934951543807983f, 2.728233814239502f, -2.273710012435913f, 1.6628460884094238f, 2.3249359130859375f, 3.8170626163482666f, 4.500376224517822f, -10.710893630981445f, 11.639716148376465f, -4.631977558135986f, 14.680305480957031f, 4.689661979675293f, -21.389917373657227f, 5.577933311462402f, 10.157344818115234f, 1.3816673755645752f, -9.77694034576416f, -1.59405517578125f, -5.5923357009887695f, 4.891682147979736f, -7.728332996368408f, -5.265920639038086f, 8.699220657348633f, -16.415210723876953f, 15.636664390563965f, -9.588492393493652f, -0.5692959427833557f, 2.7607104778289795f, 1.7100213766098022f, 5.899023532867432f, 6.637477874755859f, 1.3234360218048096f, 11.593015670776367f, 3.8447062969207764f, -12.39452075958252f, 17.31923484802246f, -4.310085296630859f, 1.3943078517913818f, 3.690248966217041f, 1.5341496467590332f, -0.16133688390254974f, -5.661871910095215f, -4.113314628601074f, 6.673490524291992f, -28.29741668701172f, -3.9005439281463623f, 5.377161979675293f, 2.418168067932129f, 14.741388320922852f, 12.219460487365723f, 2.1486709117889404f, -0.9644702672958374f, 2.6650185585021973f, -5.7412190437316895f, -11.613107681274414f, -4.728487014770508f, 4.203025817871094f, 4.285702705383301f, -6.733584880828857f, -1.751605749130249f, 1.8439090251922607f, 3.607456684112549f, -7.146326541900635f, 10.433805465698242f, 2.1755292415618896f, 7.192582607269287f, 2.093909502029419f, -12.907740592956543f, 10.749999046325684f, -0.37183743715286255f, -13.879500389099121f, -2.7690041065216064f, -0.5802304744720459f, -2.454869508743286f, 6.290034770965576f, -1.3150807619094849f, 1.3150523900985718f, -7.275416851043701f, -4.452425479888916f, 13.685830116271973f, -10.66769790649414f, 1.0413782596588135f, -11.05685043334961f, 18.703815460205078f, -1.3163670301437378f, 4.386673450469971f, 6.897695541381836f, -10.802721977233887f, -1.752117395401001f, 5.392110824584961f, -6.665993690490723f, 6.647964000701904f, 6.248779296875f, -8.761367797851562f, 3.014192581176758f, -12.200126647949219f, -4.44820499420166f, -6.547129154205322f, 5.838053226470947f, -3.972179889678955f, 4.548210620880127f, -0.38630470633506775f, 0.557646632194519f, -7.725645065307617f, -3.0316765308380127f, -10.36661148071289f, 9.558512687683105f, 1.3280283212661743f, -0.6749752759933472f, 2.5804808139801025f, 14.323273658752441f, 3.224461317062378f, -11.820540428161621f, -3.7925498485565186f, -8.334758758544922f, 0.4022352397441864f, 7.823636531829834f, -3.6518750190734863f, 2.0640387535095215f, -2.0144593715667725f, 1.261433482170105f, -12.39039421081543f, 1.648199200630188f, -8.816394805908203f, -6.666141986846924f, 4.299948692321777f, 11.934805870056152f, -9.07494831085205f, 8.816624641418457f, -7.657290935516357f, -1.1243833303451538f, 3.259331464767456f, 2.445486545562744f, 0.27485233545303345f, 9.906669616699219f, 10.861600875854492f, 1.530747413635254f, 3.6094765663146973f, 6.081133842468262f, -6.803634166717529f, 9.30950927734375f, -7.500870704650879f, -0.20945091545581818f, -4.56763219833374f, 9.062397956848145f, -3.900595188140869f, 0.46193817257881165f, -4.673060417175293f, -0.03734937310218811f, -7.745742321014404f, -1.5532246828079224f, -0.17770543694496155f, 7.410264492034912f, -2.503795862197876f, -4.5144267082214355f, 1.2131376266479492f, 1.3187705278396606f, -1.5181187391281128f, 1.0714176893234253f, 8.04094409942627f, 0.9295154213905334f, 4.887729167938232f, 4.386360168457031f, 2.0568528175354004f, -11.794127464294434f, -8.06806755065918f, 1.753397822380066f, -17.30348777770996f, -9.977322578430176f, 1.4055453538894653f, -3.8515894412994385f, -5.236893653869629f, 14.334654808044434f, -2.8135969638824463f, 12.197455406188965f, 5.414585113525391f, -3.6971359252929688f, 3.15084171295166f, -5.809675216674805f, 1.7624473571777344f, -1.7521796226501465f, -4.359520435333252f, -6.7524309158325195f, -6.430643558502197f, 4.465829372406006f, 5.363595008850098f, -0.05738918483257294f, -1.2110704183578491f, -12.731424331665039f, 7.597854137420654f, 5.500415325164795f, -6.625348091125488f, -4.435070991516113f, 16.017005920410156f, 6.0957231521606445f, 3.6745896339416504f, 18.56254005432129f, -6.415927886962891f, 7.9389967918396f, 2.5001425743103027f, 13.855697631835938f, 3.3746798038482666f, -13.604127883911133f, 0.829696774482727f, -0.6029493808746338f, 5.70452880859375f, 2.657468795776367f, -2.0609846115112305f, -5.867642402648926f, 9.778928756713867f, 4.533025741577148f, 5.282003402709961f, -1.0826342105865479f, -6.271111965179443f, 0.7187356352806091f, 8.167415618896484f, -17.741748809814453f, 5.170361518859863f, 1.9452539682388306f, 0.710178554058075f, 15.03416633605957f, 5.8055901527404785f, 9.560342788696289f, -7.244373798370361f, -7.103370189666748f, 6.701330184936523f, -13.6785306930542f, -7.435286998748779f, -1.4204037189483643f, 3.6168243885040283f, -1.2848598957061768f, 10.019153594970703f, -5.542983531951904f, 7.791118621826172f, 10.698689460754395f, 0.5794566869735718f, -5.135282039642334f, -0.725027859210968f, 6.094788551330566f, 3.9960646629333496f, 9.213926315307617f, -1.1690841913223267f, -11.310675621032715f, 4.797519207000732f, 2.9180209636688232f, 17.760066986083984f, -3.3779516220092773f, -5.8081278800964355f, 11.280855178833008f, -4.242335319519043f, -9.20323657989502f, 5.160378456115723f, 3.6303231716156006f, 8.061834335327148f, 0.7347631454467773f, 10.492819786071777f, -0.8277621865272522f, 15.720081329345703f, 6.086870193481445f, 2.3116986751556396f, -8.257061958312988f, -11.12254524230957f, -4.629530906677246f, -1.0439260005950928f, 2.503861665725708f, 4.953763484954834f, -7.421053886413574f, -4.31266975402832f, -0.7821304202079773f, -9.519991874694824f, -5.7066450119018555f, 2.1456873416900635f, 0.6137762665748596f, 9.2358980178833f, -4.393798351287842f, 0.7774427533149719f, -6.750569820404053f, 6.697755813598633f, 1.7473117113113403f, -5.084836959838867f, -0.3822762370109558f, 3.1108381748199463f, 1.4163764715194702f, -2.868486166000366f, -2.0308327674865723f, 12.085240364074707f, 0.32344046235084534f, -8.123509407043457f, -8.63557243347168f, 0.37889865040779114f, 5.865474700927734f, -2.006664752960205f, -4.2740278244018555f, -3.2797470092773438f, 0.7500452995300293f, 6.715641021728516f, 1.116511583328247f, -5.29896879196167f, 15.738234519958496f, -1.072593331336975f, 7.1444854736328125f, -0.17794132232666016f, 0.2486240416765213f, 10.500697135925293f, 5.418370246887207f, -2.850708484649658f, 5.028833866119385f, -1.3049635887145996f, -3.1064016819000244f, 7.260385036468506f, -0.2315407395362854f, 13.63786506652832f, -3.968128204345703f, 5.425351619720459f, 5.989855766296387f, 0.4256654977798462f, -11.24907112121582f, -13.982461929321289f, 3.0191447734832764f, -2.6573190689086914f, 0.18543148040771484f, 2.4465410709381104f, 7.224967002868652f, 9.639179229736328f, 11.72110366821289f, -14.941984176635742f, 2.158461809158325f, -9.967697143554688f, 7.731217384338379f, 0.2547948360443115f, 8.561541557312012f, -1.320813775062561f, -6.716103553771973f, -13.876091003417969f, -10.98348331451416f, 4.823574066162109f, 5.995814323425293f, 13.67184829711914f, -3.9956705570220947f, 2.217562675476074f, 8.181840896606445f, 11.416173934936523f, -7.419627666473389f, -2.8480899333953857f, -3.8081743717193604f, -9.326810836791992f, 2.7183923721313477f, 3.146589994430542f, -3.0829880237579346f, -5.814912796020508f, 8.27266788482666f, -2.7244033813476562f, 1.7461737394332886f, 12.851394653320312f, 6.015487194061279f, -11.382539749145508f, -10.727778434753418f, 1.8263652324676514f, 4.965327262878418f, -4.425846576690674f, 1.1692211627960205f, 0.7969360947608948f, 0.08354203402996063f, 6.083127975463867f, 5.9863152503967285f, -6.197132587432861f, -0.453395277261734f, -1.288467288017273f, -6.85847806930542f, 6.487435340881348f, -5.534632682800293f, 13.624343872070312f, 1.5271718502044678f, -1.4509201049804688f, -12.175165176391602f, -9.207465171813965f, 9.6865873336792f, -4.8576860427856445f, 5.351866245269775f, -8.007092475891113f, -1.1575658321380615f, 13.337957382202148f, 11.559327125549316f, -15.20022201538086f, 0.6360299587249756f, -16.20244789123535f, 7.8914031982421875f, 7.922462463378906f, 0.49626535177230835f, 0.6453051567077637f, 17.201900482177734f, 4.8184661865234375f, -9.721735954284668f, -9.466944694519043f, -1.581121563911438f, -3.897400379180908f, 13.486580848693848f, -2.706510305404663f, 2.2266478538513184f, -10.676257133483887f, -8.26251220703125f, -3.0972604751586914f, 6.262456893920898f, -7.870155334472656f, -13.106574058532715f, 1.9502202272415161f, 7.944813251495361f, 12.406501770019531f, -14.935750007629395f, 14.834214210510254f, -6.443024635314941f, -7.034519195556641f, 19.392358779907227f, -6.4235076904296875f, 5.952810287475586f, 4.202585697174072f, -6.166481018066406f, 6.54275369644165f, 4.269073009490967f, -4.20452356338501f, -11.693081855773926f, 0.5500400066375732f, -6.326947212219238f, -8.8986234664917f, -15.315683364868164f, 11.211211204528809f, -3.6820297241210938f, -4.241352081298828f, -0.7907126545906067f, 9.467594146728516f, 3.63688588142395f, -3.874994993209839f, -9.040021896362305f, 0.5441036224365234f, 14.560470581054688f, -4.352148532867432f, 2.122553586959839f, 7.5770955085754395f, 10.166125297546387f, -2.813676118850708f, -5.165591716766357f, -10.61868953704834f, 2.6967086791992188f, 1.273497462272644f, 16.388473510742188f, 12.926139831542969f, -18.8586483001709f, -2.5625576972961426f, -1.4167602062225342f, 3.1684563159942627f, -0.9735271334648132f, -12.724577903747559f, -9.471477508544922f, 1.9890105724334717f, -5.327760696411133f, 4.15165901184082f, -0.2690247595310211f, -5.091151237487793f, -8.251236915588379f, -13.286526679992676f, 11.11539363861084f, -2.1142094135284424f, -8.37338924407959f, -4.229764461517334f, 2.3841516971588135f, 9.676994323730469f, -16.521343231201172f, 21.009220123291016f, -14.100972175598145f, -18.230031967163086f, -2.3318724632263184f, 6.4106621742248535f, 5.1233930587768555f, 3.14448618888855f, -2.510714292526245f, 4.122291088104248f, -16.625221252441406f, -0.8044806122779846f, -7.060884952545166f, 15.047514915466309f, 1.5442057847976685f, -1.132941484451294f, 1.3845758438110352f, 5.088216781616211f, 5.57691764831543f, -5.411052703857422f, 15.526586532592773f, -8.33289623260498f, 2.9822065830230713f, -1.6032358407974243f, -16.77972984313965f, -8.319680213928223f, 1.4303581714630127f, 16.107824325561523f, -0.7335758805274963f, -10.483888626098633f, 6.441460132598877f, 4.506333827972412f, 2.4685215950012207f, 4.076876640319824f, -11.192659378051758f, -4.523487091064453f, -7.707409858703613f, 11.638382911682129f, -8.234586715698242f, 3.4906229972839355f, -9.156494140625f, -2.1987483501434326f, -14.089797019958496f, -0.26744771003723145f, 9.441929817199707f, 9.555951118469238f, 3.5714211463928223f, -13.786009788513184f, 2.148815870285034f, -6.62591028213501f, 7.7097487449646f, -1.0936174392700195f, 6.26248836517334f, 4.983580589294434f, -14.675965309143066f, -6.113205909729004f, 6.947558879852295f, -8.564501762390137f, 12.769579887390137f, -2.9595448970794678f, -2.199409008026123f, 4.292818069458008f, 0.014189623296260834f, -13.7341890335083f, 9.055596351623535f, 9.658831596374512f, 11.737518310546875f, -1.8660013675689697f, -7.643200397491455f, -7.508113384246826f, 8.38414192199707f, 5.401965618133545f, 13.07395076751709f, 3.3191146850585938f, 4.307673931121826f, 2.2782421112060547f, 1.9879189729690552f, -8.233000755310059f, -9.047646522521973f, 7.657324314117432f, 4.247537612915039f, 11.054131507873535f, -4.79965353012085f, -0.3028814196586609f, 3.658170461654663f, 6.45146369934082f, -2.8840818405151367f, 2.958085536956787f, 0.6611911654472351f, -15.455345153808594f, 14.44206428527832f, -1.5460273027420044f, 7.024033546447754f, -4.981692790985107f, -1.476488709449768f, -11.16106128692627f, 5.7634100914001465f, -4.065712928771973f, -7.169682502746582f, 6.687571048736572f, 9.344539642333984f, 0.3659030795097351f, 9.137345314025879f, -2.647665023803711f, -1.841318964958191f, 5.127373695373535f, -10.907052993774414f, 12.13635540008545f, 7.434513568878174f, -4.468003273010254f, 3.7723612785339355f, 6.410868167877197f, 4.488480091094971f, -4.279177665710449f, 7.486018180847168f, 2.041238784790039f, 0.8295110464096069f, 6.553818702697754f, 0.889700710773468f, 4.766284465789795f, -10.556770324707031f, -4.710302352905273f, -4.570557594299316f, 2.7720491886138916f, -0.9912987351417542f, -14.845259666442871f, 9.316060066223145f, 1.0089187622070312f, 8.025344848632812f, -1.9029171466827393f, 7.934247016906738f, -9.479381561279297f, 2.46075439453125f, 4.621790409088135f, -9.061838150024414f, 5.576177597045898f, 1.5385044813156128f, -0.02164754830300808f, 3.5387630462646484f, 8.792508125305176f, 11.135085105895996f, 0.09514058381319046f, 2.4890284538269043f, 6.469166278839111f, -12.828638076782227f, -11.913107872009277f, -12.835579872131348f, -0.7011198997497559f, 7.332045555114746f, 8.766676902770996f, -5.067351341247559f, -0.9870837926864624f, -12.046849250793457f, -8.263047218322754f, 7.80837869644165f, -2.4639174938201904f, 4.959366798400879f, 1.3135488033294678f, 0.6050440073013306f, 4.289178848266602f, -1.3765535354614258f, -9.210429191589355f, -2.441307544708252f, -13.704766273498535f, 1.7074304819107056f, 8.250596046447754f, -9.176819801330566f, -2.5442323684692383f, -10.301264762878418f, -9.052530288696289f, -10.033407211303711f, -6.968941688537598f, -2.9384915828704834f, 11.104891777038574f, 12.18455696105957f, -1.7053841352462769f, -10.236660957336426f, 13.148673057556152f, 5.770162105560303f, -14.608895301818848f, -3.367353677749634f, -1.997019648551941f, 0.6409828662872314f, 10.021282196044922f, 5.919330596923828f, -1.6916216611862183f, 8.11469841003418f, 15.174899101257324f, 1.4809973239898682f, 7.514713764190674f, -5.768565654754639f, 2.8586220741271973f, 0.785350501537323f, 6.117953300476074f, 12.343249320983887f, 11.014166831970215f, -3.0343480110168457f, -3.784940004348755f, -0.5094264149665833f, -3.813234567642212f, -1.6921602487564087f, 9.909290313720703f, 4.26685905456543f, -0.35572367906570435f, 13.063310623168945f, 0.9254683256149292f, -0.6383225321769714f, -4.169843673706055f, -0.2921450138092041f, -9.779097557067871f, 0.32207998633384705f, 0.46088171005249023f, -1.499398112297058f, -11.92739486694336f, 6.62190055847168f, 16.76767349243164f, 7.850483417510986f, -6.002956867218018f, -6.0743865966796875f, 8.590424537658691f, -8.350720405578613f, -7.048295497894287f, 8.3378267288208f, 10.439396858215332f, 4.051339149475098f, 7.447882652282715f, 2.115618944168091f, -10.832535743713379f, 3.544856071472168f, -10.704363822937012f, -1.747441053390503f, 3.1754229068756104f, -4.96966028213501f, -10.882390975952148f, 2.8613874912261963f, -14.762873649597168f, -3.866286516189575f, -9.634818077087402f, -4.488661289215088f, -16.146400451660156f, -8.561323165893555f, 10.843777656555176f, 7.5432963371276855f, 8.12264347076416f, -7.4483113288879395f, -10.192276000976562f, -0.18046212196350098f, 11.086979866027832f, 28.458358764648438f, -5.693641662597656f, 6.983304500579834f, 4.484742164611816f, -7.9314494132995605f, 5.449891567230225f, 3.5121002197265625f, -10.800859451293945f, 2.2634434700012207f, -10.603031158447266f, 6.369093894958496f, 21.34639549255371f, 5.682496070861816f, -6.500676155090332f, -3.933079719543457f, 9.044882774353027f, -1.2291513681411743f, 2.2405006885528564f, 7.146441459655762f, -5.252052307128906f, 13.324518203735352f, -2.54026198387146f, -4.696802139282227f, -1.9487093687057495f, 1.8994616270065308f, -12.246219635009766f, -6.6798505783081055f, -7.5826520919799805f, -4.574042320251465f, -1.7202016115188599f, 2.292769193649292f, 10.378962516784668f, 6.300224781036377f, 6.089559555053711f, 2.338552713394165f, 10.752300262451172f, -9.21757984161377f, 3.5770480632781982f, 1.6553243398666382f, -5.025013446807861f, -1.0492916107177734f, 2.3430588245391846f, -2.3533082008361816f, 13.465797424316406f, 8.130950927734375f, -17.104875564575195f, 3.76094913482666f, 3.0935189723968506f, 1.6557022333145142f, -1.1840074062347412f, 13.991741180419922f, 18.070926666259766f, -1.3191888332366943f, -5.403342247009277f, 3.2503833770751953f, 11.285150527954102f, 9.816794395446777f, -5.216005802154541f, -1.5604654550552368f, 6.930087089538574f, 2.469757556915283f, 0.5377483367919922f, -6.6069655418396f, -2.20501971244812f, -7.6572394371032715f, -12.394196510314941f, 4.9394073486328125f, -18.88083267211914f, 16.13848114013672f, 2.5295422077178955f, -6.287371635437012f, 3.2773492336273193f, 3.9482178688049316f, 1.5670056343078613f, -0.5477782487869263f, -9.703927993774414f, 7.892303466796875f, 16.467126846313477f, 1.2177623510360718f, -3.8086254596710205f, -0.10376394540071487f, 2.0969502925872803f, -13.377406120300293f, 11.43333625793457f, 0.33095118403434753f, -4.543916702270508f, -8.336254119873047f, 10.293583869934082f, -5.526627063751221f, 7.208284854888916f, 1.3287492990493774f, -0.00872433464974165f, -4.405787467956543f, 24.910831451416016f, 5.813371658325195f, -2.3747384548187256f, -1.58162522315979f, 7.138689041137695f, -0.8299044370651245f, 1.7968316078186035f, -0.16036289930343628f, 7.750216007232666f, -7.1681060791015625f, -1.6841415166854858f, 8.755945205688477f, -6.4495062828063965f, -3.929804801940918f, -11.690154075622559f, 5.407276630401611f, -19.410442352294922f, -7.46274471282959f, -6.528491497039795f, 11.476672172546387f, -10.896162986755371f, 9.078445434570312f, 4.966217994689941f, 2.7781200408935547f, 8.466453552246094f, 6.95072078704834f, -14.296183586120605f, 5.738059997558594f, -0.7061849236488342f, -3.4425599575042725f, -0.2813775837421417f, 4.374084949493408f, -2.8933258056640625f, -10.747404098510742f, -2.676332473754883f, 3.489074230194092f, 9.93961238861084f, 5.222956657409668f, -17.090818405151367f, 1.7347217798233032f, 13.138232231140137f, 8.740431785583496f, 6.423698425292969f, 1.796844244003296f, 2.321765661239624f, 6.327892303466797f, 0.3434549570083618f, 4.261251926422119f, 8.633129119873047f, -9.238373756408691f, 2.2492949962615967f, 2.5688412189483643f, -10.994841575622559f, -2.4493515491485596f, -3.7998733520507812f, -7.384821891784668f, -3.2261850833892822f, 17.528900146484375f, 3.3615753650665283f, 9.745475769042969f, -8.878006935119629f, 5.870270252227783f, -3.883683919906616f, -0.0326266773045063f, 7.409327030181885f, 10.064127922058105f, 17.231494903564453f, -4.299467086791992f, 0.6162081956863403f, -3.9329636096954346f, 3.457488775253296f, 1.6008586883544922f, 14.369483947753906f, -7.038842678070068f, 2.9360132217407227f, -11.51530933380127f, -6.849390029907227f, 13.802511215209961f, -3.17645263671875f, 8.121315002441406f, -4.379727840423584f, -0.7692107558250427f, -7.983859539031982f, -2.9673516750335693f, -16.916322708129883f, -16.98297882080078f, -10.993241310119629f, 6.7799153327941895f, -8.828645706176758f, -8.281076431274414f, 0.38319116830825806f, -6.442864418029785f, 5.842170715332031f, 2.7757022380828857f, -0.3426856994628906f, 11.318920135498047f, 9.262431144714355f, -17.823097229003906f, -11.620147705078125f, 0.7059615254402161f, 4.266175746917725f, 13.950201988220215f, -9.230637550354004f, -0.817978024482727f, 7.5885210037231445f, 15.164800643920898f, 7.9170756340026855f, -8.626565933227539f, -11.744421005249023f, -8.501404762268066f, -5.230059623718262f, -4.577203273773193f, -2.578476667404175f, 4.45864725112915f, 11.191590309143066f, 0.7849674224853516f, 2.8678131103515625f, 19.044151306152344f, -5.121172904968262f, 1.8655592203140259f, 5.731597900390625f, -4.264453411102295f, 0.31350159645080566f, 16.386018753051758f, -9.369375228881836f, -7.727303981781006f, -5.3810529708862305f, 4.404619216918945f, -5.699888229370117f, -5.798112869262695f, 0.13061971962451935f, 2.239311456680298f, 16.179100036621094f, -7.417810440063477f, 2.5456221103668213f, -3.236337900161743f, 1.1458196640014648f, -0.1307709664106369f, -0.8587997555732727f, -4.094841480255127f, -7.875479698181152f, -2.913313865661621f, 3.096707344055176f, 1.1844000816345215f, 11.496950149536133f, 4.843856334686279f, -4.238936424255371f, -1.1538751125335693f, -3.270214080810547f, 11.011395454406738f, 6.255121231079102f, 2.79670786857605f, -2.360741376876831f, 6.709340572357178f, -0.9215971827507019f, 4.753165245056152f, 11.404985427856445f, -11.04754638671875f, 5.086482048034668f, -1.1718279123306274f, 1.6178005933761597f, -1.7486406564712524f, -2.919844627380371f, 7.490941047668457f, -0.875328004360199f, -5.549253940582275f, -8.682398796081543f, 12.354720115661621f, -6.518106937408447f, 3.3926303386688232f, -7.628448486328125f, -0.5480865240097046f, 2.757507085800171f, -5.0333638191223145f, -12.717732429504395f, 9.747604370117188f, -14.137182235717773f, 13.830326080322266f, -8.072053909301758f, 9.366421699523926f, 4.771058082580566f, -3.8767879009246826f, 12.098913192749023f, -1.3983639478683472f, -2.516928195953369f, 17.085342407226562f, 0.700747013092041f, -4.593225479125977f, 3.2615785598754883f, 4.119733810424805f, 8.652438163757324f, 7.159962177276611f, -6.102161407470703f, -7.0815653800964355f, -4.8683037757873535f, -8.63952350616455f, -3.9259283542633057f, 9.596209526062012f, -9.913541793823242f, 3.975876569747925f, -3.891753911972046f, 5.650779724121094f, -6.815282344818115f, 2.731234550476074f, -0.2151940017938614f, -4.856814384460449f, 0.45669957995414734f, -7.445348262786865f, 9.566651344299316f, 1.5104864835739136f, 11.227133750915527f, -0.11537183076143265f, 4.145509719848633f, 4.6820244789123535f, 6.9148335456848145f, 4.9869384765625f, 6.348864555358887f, 0.12999588251113892f, -12.564311981201172f, -4.362220764160156f, -2.023157835006714f, -5.578927516937256f, 10.822314262390137f, -8.657614707946777f, 11.476557731628418f, -7.976850509643555f, 8.056036949157715f, 3.8919501304626465f, -3.0189290046691895f, 0.370013564825058f, 7.211360454559326f, 6.979368209838867f, -5.752582550048828f, -18.53067398071289f, -0.06240836903452873f, 6.615059852600098f, 14.322349548339844f, 7.855193614959717f, -2.3387138843536377f, -7.839303970336914f, -1.904661774635315f, 1.820509672164917f, 0.5016192197799683f, -14.7440767288208f, 13.704774856567383f, 11.45584774017334f, 10.099801063537598f, 0.022373056039214134f, -2.2562215328216553f, -3.3610339164733887f, -18.4713134765625f, -10.077775955200195f, 8.24351978302002f, -11.9453125f, 1.6951130628585815f, -6.876453399658203f, -9.18753433227539f, -3.261322498321533f, -5.389860153198242f, 1.553828239440918f, -3.9102296829223633f, 6.239824295043945f, 0.8332391381263733f, -0.4328247308731079f, -8.079927444458008f, 3.283322334289551f, 2.487851619720459f, 3.38568115234375f, 1.1550346612930298f, 5.795931339263916f, -3.827712059020996f, -17.76996612548828f, -1.3455630540847778f, -8.596356391906738f, -1.9443339109420776f, -3.347691535949707f, -3.8022572994232178f, -1.9706828594207764f, -8.254030227661133f, 4.786689758300781f, -9.022886276245117f, -10.170120239257812f, -7.9390668869018555f, -5.307765960693359f, -1.4348095655441284f, -4.2725114822387695f, 1.155957579612732f, 7.248950004577637f, 1.4605406522750854f, 2.4984130859375f, -4.926766395568848f, -1.360730767250061f, -3.5312368869781494f, -2.6760661602020264f, 13.249751091003418f, -2.3633241653442383f, 0.16462773084640503f, 2.9756968021392822f, -0.7048916220664978f, 3.0361123085021973f, 2.4167332649230957f, -18.724096298217773f, -2.668437957763672f, 4.492554664611816f, -1.228598952293396f, 4.189570426940918f, 5.219515323638916f, 1.4071485996246338f, -9.033429145812988f, 5.843366622924805f, 9.23009967803955f, -1.3791900873184204f, 2.5881869792938232f, 6.0501861572265625f, 10.04381275177002f, -6.157512187957764f, 7.24165678024292f, -4.1937384605407715f, -8.702505111694336f, 4.425203800201416f, -11.129894256591797f, -3.9875574111938477f, 10.64867877960205f, 5.056559085845947f, -10.747085571289062f, 1.264474630355835f, 7.634984016418457f, -16.58453369140625f, 8.581056594848633f, 6.325976848602295f, 2.503847360610962f, -1.0785939693450928f, 9.530458450317383f, -6.161625862121582f, -7.124795436859131f, 7.468658447265625f, -3.5646467208862305f, -2.393043041229248f, 10.85168743133545f, -1.2992841005325317f, 1.1869677305221558f, 12.415637969970703f, -0.384156733751297f, 1.6292704343795776f, -9.484331130981445f, -1.8839619159698486f, 7.0338287353515625f, -7.619279861450195f, 1.280126690864563f, -3.183976888656616f, -3.0420706272125244f, -3.4128353595733643f, -11.282638549804688f, -2.9777684211730957f, 0.18747295439243317f, -9.763144493103027f, 5.693328857421875f, 6.713829040527344f, 6.432366847991943f, 6.775195121765137f, 15.871235847473145f, -5.289862632751465f, 4.2354960441589355f, -2.9994354248046875f, -1.6106607913970947f, -8.912858009338379f, -9.295467376708984f, -8.503449440002441f, -4.607176303863525f, -1.5842245817184448f, 0.7097844481468201f, 10.443241119384766f, -0.1091175451874733f, -0.7141821384429932f, 0.6914375424385071f, -3.269829750061035f, -8.911444664001465f, -1.1965527534484863f, -1.024635910987854f, 3.5703232288360596f, -1.7879712581634521f, -3.1247658729553223f, -3.874403476715088f, -5.389334678649902f, 15.912910461425781f, -1.5096516609191895f, 10.253766059875488f, 5.733814716339111f, 0.046457380056381226f, -4.8697710037231445f, -6.196042537689209f, 26.0557861328125f, 6.26172399520874f, -6.333352088928223f, 3.3930180072784424f, -10.041016578674316f, -4.201577663421631f, -17.489336013793945f, -7.753904342651367f, -3.3036975860595703f, 9.259572982788086f, 5.622028350830078f, 7.418932914733887f, -4.166983127593994f, 3.4362385272979736f, 7.763801574707031f, 10.344937324523926f, 11.575509071350098f, -6.547769069671631f, 7.228475570678711f, 0.27201998233795166f, 11.231034278869629f, 15.090219497680664f, -3.0461761951446533f, 2.7282047271728516f, 0.22180375456809998f, 0.6711186170578003f, 7.804783821105957f, 1.4017285108566284f, 8.923149108886719f, -12.729116439819336f, 7.182803153991699f, 3.5145866870880127f, -2.1636273860931396f, -22.26779556274414f, -11.131207466125488f, -3.5905392169952393f, 25.306209564208984f, -4.522180557250977f, 8.663426399230957f, -2.190467357635498f, -9.864779472351074f, 0.2012891173362732f, -6.670775890350342f, -4.750314712524414f, 11.050165176391602f, -5.542118549346924f, 9.413408279418945f, 2.308258295059204f, 6.050876617431641f, 5.7319722175598145f, 3.2697877883911133f, -5.117337226867676f, 6.679595947265625f, 3.9800209999084473f, -13.86974048614502f, 4.446991920471191f, -8.73182201385498f, -4.240108966827393f, -3.3107078075408936f, 5.727823734283447f, -1.7871451377868652f, -1.1543947458267212f, 3.274351119995117f, 8.465099334716797f, 4.256385326385498f, 1.6312638521194458f, -0.4266124665737152f, -10.066319465637207f, -1.1442700624465942f, -13.979047775268555f, 9.38019847869873f, 10.103825569152832f, 1.44222891330719f, -12.20266056060791f, -4.161468505859375f, 5.828897476196289f, 3.763794422149658f, 3.17692494392395f, -7.616631507873535f, -4.522240161895752f, 1.3924592733383179f, 3.9834253787994385f, 7.975732326507568f, -5.482377529144287f, -10.70419979095459f, -1.8837058544158936f, -5.173501968383789f, -0.19477716088294983f, -3.4711830615997314f, 10.08415699005127f, -1.9378299713134766f, 5.39734411239624f, -4.766819953918457f, -7.339071750640869f, 4.702202796936035f, 6.930264949798584f, 14.5244779586792f, 1.373946189880371f, -7.17212438583374f, 4.260952949523926f, 13.407155990600586f, 1.6655051708221436f, 9.432223320007324f, 10.111971855163574f, -19.2701473236084f, 7.098327159881592f, -10.078179359436035f, 2.730682373046875f, -6.775233268737793f, 2.490149974822998f, 6.198461532592773f, 0.2401915341615677f, 13.640817642211914f, 6.729548454284668f, -5.323953151702881f, 7.341331958770752f, 0.6824727058410645f, -1.65504789352417f, -13.688631057739258f, 4.6387176513671875f, 3.3633487224578857f, -13.678863525390625f, 5.560359477996826f, -2.417947769165039f, 6.356764316558838f, 11.856796264648438f, -4.349381446838379f, -13.190492630004883f, 0.9404691457748413f, -0.07949790358543396f, 5.1438212394714355f, -5.610644817352295f, 9.887959480285645f, -10.704916000366211f, -6.3003129959106445f, 1.457470178604126f, -5.060678005218506f, 3.0186398029327393f, 0.4794176518917084f, 5.93607759475708f, 5.506703853607178f, -1.0873303413391113f, 2.6558244228363037f, 5.770946502685547f, -4.9762678146362305f, 12.409749031066895f, -7.4365234375f, 8.309518814086914f, 0.3051859140396118f, -6.369776248931885f, -6.938343048095703f, 14.529619216918945f, -8.19320011138916f, 4.215144157409668f, -8.835214614868164f, 2.212118148803711f, 7.021667003631592f, 1.2244818210601807f, -6.6155571937561035f, -5.983504295349121f, 5.949681758880615f, -9.470444679260254f, 1.7039216756820679f, 4.452876567840576f, 3.5223419666290283f, 1.386846661567688f, 12.392443656921387f, 3.4754412174224854f, -3.757160186767578f, 2.243192672729492f, 1.5498485565185547f, -5.969877243041992f, -9.53218936920166f, -0.9648003578186035f, -4.1782612800598145f, -8.63139533996582f, 2.888253688812256f, -4.9197845458984375f, -2.6188461780548096f, -0.34100860357284546f, 1.08323073387146f, 0.20715828239917755f, 0.17625738680362701f, 0.8104458451271057f, 6.235602378845215f, 1.1863518953323364f, 2.2748658657073975f, -2.9206295013427734f, -7.237782001495361f, 7.247722625732422f, -4.8302412033081055f, -16.587038040161133f, 3.3862411975860596f, 5.526927471160889f, -2.761680841445923f, -5.178590774536133f, 12.247193336486816f, -13.684733390808105f, -5.715342998504639f, -8.004195213317871f, 11.39901065826416f, -8.808146476745605f, -11.446958541870117f, -5.106409549713135f, -2.114779472351074f, 0.827870786190033f, 7.99212121963501f, 16.132234573364258f, -7.876288414001465f, 4.045842170715332f, 4.89345121383667f, 7.8902387619018555f, 2.6189887523651123f, 1.4187779426574707f, -4.95357608795166f, 1.9183154106140137f, -10.423624038696289f, 5.698985576629639f, -10.64306640625f, 8.366253852844238f, -6.523881435394287f, -12.701069831848145f, 18.55314064025879f, -17.26905059814453f, -2.0632481575012207f, 8.771729469299316f, 6.320028781890869f, 2.5256295204162598f, 14.239582061767578f, 3.1535520553588867f, 4.2085394859313965f, 0.1063476949930191f, 2.853101968765259f, 7.213879585266113f, 7.257656097412109f, -5.45920467376709f, -4.268126010894775f, -1.759271502494812f, -0.4200115203857422f, 3.3460514545440674f, 3.7233402729034424f, 1.9605106115341187f, 10.866938591003418f, -3.5412983894348145f, 0.06446467339992523f, -13.610902786254883f, -0.3217482566833496f, -13.58945083618164f, 7.48920202255249f, -4.73434591293335f, -4.7902374267578125f, 15.362975120544434f, -2.182408571243286f, 5.474664211273193f, -9.107425689697266f, 1.9057763814926147f, 5.691718101501465f, -5.837611675262451f, 0.9479477405548096f, 5.760697841644287f, 0.09852147847414017f, 4.3325629234313965f, -1.1939945220947266f, -9.364533424377441f, 5.033231735229492f, 4.058061122894287f, -13.785080909729004f, 1.9617301225662231f, -15.00662899017334f, 1.1908645629882812f, 1.5933691263198853f, -3.296618938446045f, 0.3798801302909851f, 3.9930577278137207f, 13.144612312316895f, -19.245630264282227f, 3.366441011428833f, -9.155226707458496f, -4.203514099121094f, -6.379355430603027f, 3.052100896835327f, 3.3825178146362305f, -5.03127384185791f, -20.389877319335938f, 15.295291900634766f, -17.634958267211914f, 7.198425769805908f, -26.755468368530273f, -5.668976783752441f, 9.652477264404297f, 1.5629017353057861f, -3.2165329456329346f, -2.528200626373291f, 8.87691593170166f, 1.8550244569778442f, 12.378725051879883f, 0.00554652651771903f, 5.4776787757873535f, -0.582221508026123f, -3.155843734741211f, 12.177905082702637f, -0.9750184416770935f, -0.16764011979103088f, -19.229124069213867f, 5.547354221343994f, 14.744590759277344f, 7.076948642730713f, 11.192045211791992f, 8.736725807189941f, -10.47435188293457f, -1.8428961038589478f, 8.822990417480469f, 10.079815864562988f, -9.724599838256836f, 4.329649925231934f, 22.17165756225586f, -12.209203720092773f, 6.37702751159668f, -15.575246810913086f, 4.703880786895752f, -1.969223976135254f, 0.45949697494506836f, 0.20915208756923676f, 11.847763061523438f, -3.744934558868408f, -13.487935066223145f, 3.874084234237671f, -4.578466415405273f, -7.086031436920166f, -6.233429908752441f, -6.785592555999756f, -3.3219637870788574f, 8.725717544555664f, 8.299102783203125f, 12.64601993560791f, -5.9381513595581055f, 0.5933879613876343f, 10.182705879211426f, 5.442933559417725f, 1.8527637720108032f, -4.759885787963867f, -4.977668285369873f, -7.128690719604492f, -2.2508225440979004f, -5.206679344177246f, 1.6984038352966309f, 3.4979476928710938f, -0.9469919204711914f, -5.347060680389404f, 2.9652979373931885f, -3.192228317260742f, 4.303493499755859f, 9.290297508239746f, 9.933444023132324f, -5.329427719116211f, 2.070354700088501f, 2.8292171955108643f, 13.59816837310791f, -2.8414688110351562f, 2.1585311889648438f, 5.236742973327637f, -2.824960947036743f, -18.918386459350586f, 0.8947755098342896f, -7.695527076721191f, 1.7885832786560059f, -18.260061264038086f, -15.04204273223877f, -1.8974013328552246f, -9.819128036499023f, 8.226827621459961f, -5.237205505371094f, 4.667030334472656f, -13.77105712890625f, 4.586790084838867f, -0.390918493270874f, -4.865528106689453f, -5.15276575088501f, 8.497437477111816f, 1.938669204711914f, -4.695268154144287f, 8.078354835510254f, 4.665389537811279f, -8.80441665649414f, 10.505725860595703f, -7.680267810821533f, -1.3688803911209106f, -3.3594307899475098f, 5.555975437164307f, 11.535223007202148f, -0.9209914803504944f, 2.747661590576172f, -14.198168754577637f, -2.046473503112793f, 9.80955982208252f, 0.09400451183319092f, 7.27423095703125f, 8.030008316040039f, -7.390738487243652f, 0.04207156226038933f, 6.497010231018066f, 2.9516983032226562f, -5.786115646362305f, 6.287818908691406f, 14.243786811828613f, -8.465189933776855f, -1.7876099348068237f, -5.9232072830200195f, 5.831533908843994f, -6.165143013000488f, 6.243556499481201f, -3.9629876613616943f, 10.030241966247559f, 13.016631126403809f, -6.402768611907959f, -3.428553819656372f, -2.3323075771331787f, 0.22513322532176971f, 9.516219139099121f, 8.901734352111816f, 1.7534918785095215f, 6.5073628425598145f, -11.850318908691406f, 22.981639862060547f, 0.9134898781776428f, -6.708924293518066f, 8.352168083190918f, 6.139276027679443f, -11.110394477844238f, -12.100716590881348f, 3.7225136756896973f, -6.486250400543213f, 4.79709529876709f, 3.2657382488250732f, -6.960175514221191f, 4.827823162078857f, 3.0818111896514893f, 4.233963966369629f, -1.300526738166809f, 8.41450023651123f, 27.55376625061035f, 6.323548793792725f, 1.2947889566421509f, -4.953489303588867f, 1.9106656312942505f, 1.7545992136001587f, 4.909358024597168f, -2.6168389320373535f, -13.989197731018066f, 2.96791672706604f, -15.628793716430664f, -4.646886348724365f, 2.3803651332855225f, -0.38021320104599f, 7.955515384674072f, 10.828896522521973f, -4.704763412475586f, -2.4767675399780273f, 4.345104694366455f, 9.674301147460938f, 14.051787376403809f, -1.9404326677322388f, -6.1034440994262695f, 4.96730375289917f, 8.356350898742676f, -12.492217063903809f, 5.052522659301758f, -11.288028717041016f, -18.85377311706543f, -2.1773197650909424f, 3.268491268157959f, -11.059211730957031f, 12.636406898498535f, 8.842144966125488f, 4.771931171417236f, 13.92164134979248f, -9.583444595336914f, 18.409446716308594f, -7.4875359535217285f, 24.48175621032715f, 5.571856498718262f, 3.763981819152832f, 2.6589486598968506f, -6.752713680267334f, -14.821791648864746f, -4.6671833992004395f, -7.933231830596924f, -5.372705459594727f, -16.37500762939453f, 8.258382797241211f, 4.597978115081787f, -4.5285491943359375f, -16.733186721801758f, -15.706796646118164f, -9.255148887634277f, 5.370093822479248f, -2.010138988494873f, 3.0411553382873535f, 5.09183406829834f, -0.09162905812263489f, 5.09684419631958f, 7.237029075622559f, -1.3704605102539062f, 15.3106107711792f, -9.188556671142578f, -2.035550117492676f, -11.181009292602539f, -3.858069658279419f, 0.42419710755348206f, 10.823795318603516f, 2.402636766433716f, -2.0442917346954346f, 2.90970778465271f, -10.479278564453125f, -5.484250068664551f, -5.404007434844971f, 5.897701263427734f, -10.198234558105469f, 3.919404983520508f, 11.954742431640625f, -3.368873119354248f, -2.416825532913208f, -5.485574245452881f, -1.0831294059753418f, 12.187384605407715f, 9.810623168945312f, 2.734238624572754f, -3.1333184242248535f, -5.630797386169434f, -3.6640121936798096f, -5.316211223602295f, -5.74959135055542f, -0.2944042682647705f, -23.43301010131836f, -3.242238998413086f, 3.7050588130950928f, 10.571874618530273f, 10.871262550354004f, 0.4957568347454071f, -13.565269470214844f, -0.34055426716804504f, -5.27890682220459f, -8.633820533752441f, -4.941458702087402f, -1.589221715927124f, 0.5624170899391174f, 5.252312183380127f, -6.480530261993408f, 1.477312445640564f, 3.0334291458129883f, 1.7544362545013428f, -1.811592698097229f, 2.166680097579956f, -2.746460199356079f, -14.217162132263184f, 0.9965202212333679f, -3.636343479156494f, -4.826642990112305f, -3.4294586181640625f, -13.18789291381836f, 1.1160309314727783f, 0.03090650774538517f, 4.610648155212402f, -6.316239356994629f, 5.343184947967529f, -3.5397722721099854f, 13.128379821777344f, 6.051750659942627f, 3.017526626586914f, -0.22869278490543365f, 1.0593830347061157f, -4.756152153015137f, -9.622599601745605f, -14.170649528503418f, 2.4119977951049805f, 3.543015241622925f, -6.777645111083984f, -2.0697014331817627f, -1.148790717124939f, -5.165157318115234f, -2.193115472793579f, 2.605295181274414f, -12.414362907409668f, -16.186542510986328f, -5.055665969848633f, 14.077547073364258f, -9.300932884216309f, 4.403715133666992f, -3.700418710708618f, 1.266947865486145f, 4.215131759643555f, 10.175348281860352f, -0.7335017919540405f, -6.666941165924072f, 1.7698924541473389f, 2.5551509857177734f, 14.33661937713623f, -6.243782043457031f, -7.089589595794678f, 5.264889240264893f, 2.0064051151275635f, -1.1940221786499023f, -0.8845052719116211f, 4.728024959564209f, 9.143659591674805f, -11.243470191955566f, -3.881427764892578f, 1.0975744724273682f, 5.020593166351318f, -2.509296178817749f, -0.7040889263153076f, -8.826937675476074f, 9.25462818145752f, 0.6523112058639526f, 8.413361549377441f, -9.956643104553223f, 4.320390701293945f, -7.327249050140381f, 0.7185436487197876f, 5.529834747314453f, 3.7425594329833984f, 11.977640151977539f, 0.3199857473373413f, -1.2970579862594604f, -0.6297903656959534f, -6.748528480529785f, -1.8058745861053467f, 7.02766227722168f, -14.052090644836426f, 3.328584909439087f, -11.310714721679688f, -8.255908966064453f, -1.6174887418746948f, 2.040614604949951f, -7.178774833679199f, -8.185322761535645f, -12.005358695983887f, -1.7305762767791748f, -15.079358100891113f, -8.002707481384277f, -13.309523582458496f, -4.7150774002075195f, -7.385843276977539f, 3.19026517868042f, -0.39262112975120544f, 10.768528938293457f, -12.257975578308105f, -20.101083755493164f, -10.958288192749023f, 7.338348388671875f, -5.042275905609131f, -4.23043966293335f, -4.301355361938477f, 6.800642967224121f, 0.3120475709438324f, 10.304023742675781f, -13.633195877075195f, -0.5820647478103638f, 5.642172813415527f, -7.806666851043701f, 4.420014381408691f, -11.09785270690918f, -0.6035222411155701f, -3.352290630340576f, -11.287544250488281f, -9.170281410217285f, 4.307720184326172f, 5.357173442840576f, 9.775383949279785f, -4.904134273529053f, 0.8738774657249451f, -1.9923650026321411f, -13.437588691711426f, 4.374331951141357f, -6.224425315856934f, -7.205249786376953f, -1.2282158136367798f, 16.407073974609375f, 14.745433807373047f, 10.053177833557129f, -5.026731014251709f, 13.399908065795898f, -4.700345039367676f, -2.810746908187866f, 4.868433952331543f, 14.18775463104248f, 0.07687458395957947f, -1.2327467203140259f, -1.3728224039077759f, 7.709836483001709f, 3.666210889816284f, 4.985335826873779f, -12.233044624328613f, -5.2307329177856445f, -5.411324977874756f, 5.240880012512207f, -2.543198347091675f, -21.135807037353516f, -11.415369987487793f, 5.937788486480713f, -0.01360476203262806f, -15.29384708404541f, -23.956432342529297f, 1.6906027793884277f, 3.3760499954223633f, 9.199305534362793f, 12.791433334350586f, 4.497912883758545f, 3.4790523052215576f, -2.8718466758728027f, 4.7231268882751465f, -12.663939476013184f, 8.505736351013184f, 0.8276928663253784f, -6.634997367858887f, 3.213438034057617f, -12.446330070495605f, -0.040080517530441284f, 8.142585754394531f, -5.221867084503174f, -7.002190113067627f, -0.7204623818397522f, -11.662397384643555f, -1.66923189163208f, 2.7600550651550293f, -7.987179279327393f, 1.4977517127990723f, -1.5806363821029663f, -3.7816100120544434f, 0.5635890960693359f, 1.5413968563079834f, -1.8732370138168335f, 0.9285376667976379f, -21.148822784423828f, 19.22353744506836f, -2.7432806491851807f, -14.096722602844238f, 8.600056648254395f, -8.63492488861084f, -0.5446416139602661f, -9.644774436950684f, 1.5598598718643188f, 9.11766242980957f, 11.372527122497559f, -12.296475410461426f, 7.532688617706299f, -8.911373138427734f, 6.434876441955566f, 3.617435932159424f, 4.387059688568115f, 12.811718940734863f, -9.318404197692871f, -0.4164440929889679f, -4.88645601272583f, -9.689740180969238f, -7.375895023345947f, 3.4132399559020996f, -5.05049991607666f, -4.028200626373291f, -8.181990623474121f, -7.417090892791748f, -6.058056831359863f, 4.370342254638672f, 4.167178630828857f, 10.502394676208496f, -8.585572242736816f, -7.519354820251465f, 11.942482948303223f, 13.272440910339355f, -2.757564067840576f, 1.3308485746383667f, -2.266592264175415f, -6.452311992645264f, -8.549099922180176f, -6.8195576667785645f, -6.657449722290039f, -11.0411376953125f, -21.244892120361328f, -9.757211685180664f, 3.444068670272827f, 5.706812858581543f, -1.7444701194763184f, 8.521875381469727f, -5.228854179382324f, -5.382219314575195f, -6.763889789581299f, -11.332088470458984f, -5.202573299407959f, 1.0299209356307983f, -11.423958778381348f, -1.0182582139968872f, 12.390052795410156f, -10.38305950164795f, -5.015763282775879f, 10.922207832336426f, -2.7742760181427f, 16.766313552856445f, -7.944525241851807f, -2.8002591133117676f, 10.874272346496582f, -9.857773780822754f, 2.1176915168762207f, -7.479413032531738f, -1.7775098085403442f, 8.738555908203125f, -1.012858271598816f, -0.524698793888092f, -2.0892815589904785f, 2.1344618797302246f, 2.1381497383117676f, -8.60706615447998f, -3.834965944290161f, 5.380415916442871f, -5.791232109069824f, 12.458983421325684f, -2.1414995193481445f, -6.775130271911621f, 6.628716468811035f, -6.176581382751465f, 1.7703351974487305f, 4.631689071655273f, 6.3223066329956055f, 2.441192388534546f, -5.126554012298584f, -1.0228726863861084f, 2.649749517440796f, 2.357867956161499f, -2.623882532119751f, 0.692119300365448f, 7.061679840087891f, -1.3346446752548218f, 9.300801277160645f, -2.3639140129089355f, -14.160957336425781f, 4.440097808837891f, 1.5377870798110962f, -2.905083179473877f, -14.448901176452637f, 3.674952983856201f, -3.8605237007141113f, 8.271306037902832f, 6.405794620513916f, -21.742176055908203f, -11.455822944641113f, 1.7873941659927368f, -11.738869667053223f, 8.434774398803711f, -20.15655517578125f, -1.367598056793213f, 7.51062536239624f, -7.572475910186768f, -0.1449132263660431f, 0.14903344213962555f, 1.539661169052124f, -8.20009994506836f, -7.516574382781982f, 6.837973117828369f, -5.781085968017578f, 1.294516921043396f, -9.226051330566406f, 1.2180476188659668f, -4.204068183898926f, 15.320548057556152f, -6.9747209548950195f, 10.799696922302246f, 9.24210262298584f, 9.05672550201416f, -7.968204498291016f, -4.376702308654785f, -2.1221911907196045f, 0.1324523389339447f, 4.775428771972656f, -0.22519434988498688f, -7.35526180267334f, 0.1219637468457222f, -11.087621688842773f, -7.007081985473633f, 5.306663990020752f, -16.567237854003906f, -9.150774002075195f, 19.48033332824707f, 9.253161430358887f, 8.751167297363281f, 6.719225883483887f, 5.08009672164917f, -6.268258571624756f, 0.8974515795707703f, -12.448920249938965f, 19.20334815979004f, -12.337990760803223f, -2.6060383319854736f, 17.232389450073242f, -5.7053303718566895f, -2.461069107055664f, -9.090248107910156f, -2.2077951431274414f, 5.545995235443115f, 2.147212028503418f, 8.779653549194336f, -1.8891314268112183f, 10.41473388671875f, 5.098225116729736f, 3.8328142166137695f, 4.6025872230529785f, -3.5405564308166504f, 4.929543972015381f, -15.190320014953613f, -4.475381851196289f, 4.857414722442627f, -0.2330942153930664f, -9.112611770629883f, -14.155166625976562f, -14.117636680603027f, 0.3170005977153778f, -6.148338794708252f, 2.6847426891326904f, 1.3375723361968994f, -10.719864845275879f, 19.82689666748047f, -11.773765563964844f, 1.4471533298492432f, 5.438664436340332f, 10.833401679992676f, 5.91309928894043f, -4.8504557609558105f, -0.8866681456565857f, 4.502740383148193f, 8.76336669921875f, -7.2705078125f, -2.602914810180664f, 0.4528222978115082f, 0.2632019817829132f, -3.9463186264038086f, -6.615810394287109f, -12.216948509216309f, 18.130096435546875f, -11.216374397277832f, -9.20461368560791f, 2.837886333465576f, -9.17361068725586f, -13.872835159301758f, 1.4739644527435303f, -4.641720294952393f, -6.107040882110596f, -0.7774397730827332f, 2.1887550354003906f, -3.053284168243408f, 6.492974758148193f, -8.238874435424805f, -17.8471622467041f, -5.672789096832275f, -2.3216474056243896f, -1.9701411724090576f, -7.030838966369629f, -2.107118844985962f, -0.4771590828895569f, -1.8420956134796143f, -14.038286209106445f, -4.026478290557861f, 3.295030355453491f, 4.460195064544678f, 6.017807960510254f, 7.253128528594971f, -7.15041971206665f, 0.515494167804718f, 4.030392646789551f, -3.3914072513580322f, 0.44524064660072327f, -0.19258594512939453f, -0.7080201506614685f, -2.50250244140625f, 2.334340810775757f, -11.007967948913574f, -12.750214576721191f, -7.300755977630615f, 14.395295143127441f, -3.3914880752563477f, 8.228107452392578f, 9.74307918548584f, 0.77601557970047f, 2.347871780395508f, 9.640531539916992f, -4.919435024261475f, -7.991837024688721f, 0.03702786564826965f, 0.6593180894851685f, -8.348962783813477f, 4.535181999206543f, 3.7413909435272217f, -4.0721588134765625f, 7.057863712310791f, -1.0119569301605225f, -5.5510711669921875f, -11.117044448852539f, 5.491680145263672f, 12.420897483825684f, 12.342089653015137f, -13.135498046875f, 0.531191349029541f, -3.1611971855163574f, 2.2477285861968994f, -4.986516952514648f, 7.761373996734619f, 2.0133655071258545f, 9.318339347839355f, 8.271308898925781f, 2.2258410453796387f, 7.38371467590332f, 1.7697837352752686f, 7.604837894439697f, -5.065467357635498f, 15.182759284973145f, 10.54971694946289f, 14.205101013183594f, -15.358622550964355f, -7.208074569702148f, -1.3423936367034912f, 4.829193115234375f, -6.763162612915039f, -6.093154430389404f, -3.8595669269561768f, 10.870450019836426f, 5.670471668243408f, -9.437480926513672f, 2.4107041358947754f, -6.857131004333496f, 2.4801251888275146f, -5.085203170776367f, 15.741228103637695f, -18.639524459838867f, 0.10690624266862869f, 10.266143798828125f, 0.7210645079612732f, 2.90657114982605f, 4.488924980163574f, 11.784412384033203f, 10.836816787719727f, 11.812047004699707f, 0.9760425090789795f, 6.398825645446777f, -11.69657039642334f, 9.617613792419434f, -2.4754092693328857f, 2.3830978870391846f, 4.122246742248535f, -6.237287521362305f, 6.2666802406311035f, -7.420814514160156f, -4.996058940887451f, 6.019990921020508f, 8.033538818359375f, -3.3236873149871826f, -10.9536714553833f, 6.0252366065979f, -10.10529613494873f, 10.607632637023926f, -1.0954471826553345f, -7.890513896942139f, -6.947789669036865f, 18.596343994140625f, -10.062642097473145f, 9.224631309509277f, 12.48892593383789f, 11.919086456298828f, 10.027180671691895f, 8.543699264526367f, -8.874430656433105f, -2.0335042476654053f, -14.272953033447266f, 3.9183735847473145f, -8.344069480895996f, -11.0534086227417f, 2.4750025272369385f, -0.029886547476053238f, 5.07313871383667f, -26.66839599609375f, -8.733036041259766f, -4.485158920288086f, 9.778528213500977f, 5.770241737365723f, 12.092632293701172f, 1.5609368085861206f, 11.50508975982666f, -0.6498426795005798f, -9.589741706848145f, -27.592344284057617f, -0.43285009264945984f, 11.695541381835938f, -6.151013374328613f, 5.020718097686768f, -12.249852180480957f, -2.1764540672302246f, 6.811885833740234f, 16.927019119262695f, -10.076769828796387f, -17.306066513061523f, 4.369997978210449f, 1.4093303680419922f, -2.871824026107788f, 2.7676126956939697f, -6.874851226806641f, -2.597337484359741f, -9.963288307189941f, 10.695664405822754f, 5.037891387939453f, -6.623497009277344f, 5.716047286987305f, 4.312154769897461f, 4.035192489624023f, 2.1116819381713867f, 3.357062578201294f, 1.416142225265503f, 3.6634714603424072f, -4.913036823272705f, -5.307648181915283f, 11.786884307861328f, -1.3370147943496704f, 10.858242988586426f, 8.861958503723145f, 6.27660608291626f, 12.768265724182129f, 12.327997207641602f, -8.675048828125f, -3.4217770099639893f, 7.692430019378662f, 1.7838584184646606f, 12.854020118713379f, -11.746981620788574f, -9.584903717041016f, 8.365598678588867f, 12.973992347717285f, 5.4280571937561035f, -9.301798820495605f, 11.192365646362305f, -2.602813243865967f, 6.336965084075928f, 0.5768340229988098f, -10.169051170349121f, -8.006020545959473f, 13.611660957336426f, -5.536235332489014f, 3.161245346069336f, -9.059117317199707f, -2.214181423187256f, -5.4537811279296875f, 6.635800838470459f, -12.098894119262695f, 11.331655502319336f, 13.68090534210205f, 3.1792664527893066f, 1.3622552156448364f, 2.2142395973205566f, 0.2345779687166214f, 0.5814613103866577f, 4.289124011993408f, 4.538444995880127f, -7.362942695617676f, 2.874087333679199f, 3.61981201171875f, -5.420032501220703f, 8.720592498779297f, 4.290198802947998f, 12.945777893066406f, 5.480059623718262f, -7.588292598724365f, 5.661287784576416f, -3.8283472061157227f, -8.462838172912598f, 7.285812854766846f, -10.747210502624512f, -3.003866672515869f, 7.091235637664795f, -1.580762505531311f, 8.499457359313965f, 0.9311726689338684f, 2.294097661972046f, 7.290738582611084f, -4.4237260818481445f, 13.710098266601562f, -7.216895580291748f, 2.5698392391204834f, -4.586067199707031f, -5.298479080200195f, -3.6828322410583496f, -1.1227331161499023f, 6.324790954589844f, 0.32012125849723816f, -0.4937366843223572f, -1.9536423683166504f, -9.237577438354492f, 1.0877151489257812f, 12.129581451416016f, 4.084137916564941f, -2.9383089542388916f, -5.019631862640381f, 0.6925498247146606f, 6.584831714630127f, 0.4018475115299225f, 0.21690116822719574f, 1.1919173002243042f, 2.2553939819335938f, -1.0278080701828003f, -13.67727279663086f, -1.5738617181777954f, 2.352642297744751f, 3.2829935550689697f, 0.22515998780727386f, -5.764885902404785f, -3.821402072906494f, 7.599716663360596f, 4.4758172035217285f, -18.555747985839844f, 10.015554428100586f, -5.277355194091797f, 4.857947826385498f, -4.972664833068848f, -7.69036865234375f, 0.8392868041992188f, -26.074081420898438f, -8.01816463470459f, 17.92840576171875f, 4.751870632171631f, 1.765329360961914f, -4.687397003173828f, 8.163311958312988f, 4.182957172393799f, 8.034849166870117f, 1.3160547018051147f, -0.5854548215866089f, -8.705289840698242f, -0.8203011155128479f, -3.8902640342712402f, -3.927004814147949f, 10.755240440368652f, -2.724745035171509f, -5.045247554779053f, -13.067852973937988f, 15.737813949584961f, 1.6259382963180542f, 0.018999869003891945f, 1.839993953704834f, 2.3978917598724365f, 1.3756414651870728f, 10.791954040527344f, -4.205836772918701f, 7.3124308586120605f, 3.0303399562835693f, 18.43343734741211f, 2.922717571258545f, 7.7857747077941895f, 9.202900886535645f, -11.383536338806152f, 4.063195705413818f, 4.0933308601379395f, 9.472505569458008f, 0.6550918817520142f, 1.219048261642456f, -3.383958101272583f, -13.222259521484375f, 6.909287452697754f, -11.573260307312012f, 4.1832146644592285f, 6.807440280914307f, -12.13596248626709f, -16.246288299560547f, -11.497807502746582f, 8.08572769165039f, -12.573715209960938f, -4.854974746704102f, -2.2262701988220215f, 0.8039400577545166f, 11.886690139770508f, -15.268168449401855f, -8.579140663146973f, -2.6850054264068604f, 5.244555950164795f, -6.70413064956665f, -3.2002172470092773f, 6.914629936218262f, 0.3705389201641083f, 14.895472526550293f, -6.253382205963135f, -6.998981475830078f, 4.563458442687988f, 5.477730751037598f, -9.057239532470703f, 9.500064849853516f, 4.373735427856445f, -0.7703867554664612f, 13.951043128967285f, -5.243332862854004f, 2.8435473442077637f, -6.590154647827148f, 4.319807529449463f, -2.5442638397216797f, -13.913573265075684f, -10.475417137145996f, -2.8347291946411133f, 8.493965148925781f, -10.500249862670898f, -5.360538005828857f, -4.234364986419678f, 0.10634779930114746f, -1.185592532157898f, -9.338807106018066f, 2.1675655841827393f, 3.200042486190796f, 4.2734856605529785f, 5.105844020843506f, -7.451834201812744f, -11.285500526428223f, -2.5912656784057617f, -4.90095853805542f, -7.117650508880615f, 5.817300796508789f, -2.694540023803711f, -6.723092555999756f, 3.3572030067443848f, 9.950608253479004f, -2.062587261199951f, -2.237545967102051f, 5.24613094329834f, 18.30078887939453f, -7.8479437828063965f, -2.280181407928467f, 8.82515811920166f, 0.1949726939201355f, 4.303898811340332f, -3.4921815395355225f, 16.898700714111328f, 11.57261848449707f, -2.4016921520233154f, -3.2015914916992188f, 10.793314933776855f, -6.424396514892578f, 1.5789941549301147f, 6.381648540496826f, -21.38960838317871f, 0.6480197906494141f, -13.997282028198242f, 0.6987282037734985f, -7.389518737792969f, 10.640852928161621f, -1.155595064163208f, -2.187462091445923f, -2.0233631134033203f, -2.092860698699951f, 8.194184303283691f, 12.357358932495117f, 12.201729774475098f, 5.951974868774414f, 1.862755537033081f, -4.316686630249023f, 6.706587791442871f, 5.658695220947266f, -3.7403807640075684f, 4.848812580108643f, -1.1044867038726807f, 4.073699474334717f, -6.804827690124512f, -2.227973461151123f, -14.886821746826172f, 4.097927093505859f, -5.149636745452881f, 14.20467472076416f, 10.393085479736328f, 16.598787307739258f, 20.412975311279297f, 0.4636867642402649f, -12.83670711517334f, 5.011075496673584f, 3.2573134899139404f, 2.541311502456665f, -3.704630136489868f, -7.621809482574463f, 2.2266361713409424f, -6.034920692443848f, -4.907690525054932f, 3.311673164367676f, -11.02491569519043f, 6.5720438957214355f, -13.933245658874512f, 3.847233295440674f, 12.274158477783203f, 0.9272652864456177f, -1.7610269784927368f, 0.4040965139865875f, 0.680636465549469f, -15.860685348510742f, 1.9717633724212646f, -9.28317928314209f, -2.6344847679138184f, 1.870112657546997f, -15.349178314208984f, -5.096085548400879f, -1.4747427701950073f, -8.14248275756836f, 2.2569987773895264f, -15.152320861816406f, -7.701322078704834f, 10.674273490905762f, -1.4894379377365112f, -4.1722493171691895f, -20.385786056518555f, -7.458427429199219f, 0.15474386513233185f, 6.180734634399414f, 9.458477973937988f, 3.9757118225097656f, 3.1847710609436035f, 2.5685551166534424f, -1.0029635429382324f, -0.9310358762741089f, -5.389017581939697f, 13.258286476135254f, 4.855055809020996f, 11.788922309875488f, 2.5168514251708984f, -8.065077781677246f, 5.255640983581543f, 12.24008846282959f, -13.184330940246582f, -0.7134942412376404f, -8.317489624023438f, 11.638908386230469f, -3.975881814956665f, 0.43127647042274475f, -8.266470909118652f, -12.825586318969727f, 13.573546409606934f, 8.546313285827637f, 4.274265289306641f, 3.6718971729278564f, 10.517722129821777f, 0.27425989508628845f, 8.075490951538086f, -2.1264874935150146f, 0.6136637926101685f, -5.506547451019287f, 15.231038093566895f, 0.46134939789772034f, 0.17480090260505676f, 2.040527582168579f, 3.2183902263641357f, -6.139379501342773f, -9.661615371704102f, -10.546405792236328f, -3.0357425212860107f, -3.1846868991851807f, -4.761189937591553f, -3.7215702533721924f, 3.0658512115478516f, -11.569316864013672f, -0.43601199984550476f, 2.8654866218566895f, -2.9958226680755615f, 2.4096181392669678f, -10.564579963684082f, -13.85832405090332f, -6.305012226104736f, 7.627152919769287f, 2.5408518314361572f, -10.277200698852539f, 1.9411969184875488f, -4.243484973907471f, 1.4209707975387573f, 3.253671169281006f, 1.769997000694275f, -9.947007179260254f, 3.4785919189453125f, 3.4994893074035645f, 14.112330436706543f, 17.531051635742188f, 3.3806848526000977f, 2.1774063110351562f, -14.381307601928711f, 6.56845235824585f, -11.52322769165039f, 0.6346222162246704f, 7.065883636474609f, -3.087952136993408f, 6.3684282302856445f, -6.367730140686035f, 6.013090133666992f, 4.485490322113037f, 3.8508636951446533f, -11.656087875366211f, 0.451434463262558f, 7.0519280433654785f, 6.291365623474121f, 10.597298622131348f, 7.171876430511475f, 1.4291812181472778f, -8.897068977355957f, 0.9057914614677429f, 1.817928671836853f, -3.542912006378174f, 15.661563873291016f, -3.5200037956237793f, -1.275672435760498f, 11.505273818969727f, -5.713345527648926f, -13.151139259338379f, 1.984048843383789f, 24.558940887451172f, -0.4125036895275116f, 7.0134735107421875f, 14.921035766601562f, 3.6945953369140625f, 15.289679527282715f, -12.831208229064941f, -7.242607593536377f, -14.355047225952148f, -3.520359992980957f, 17.282093048095703f, -1.093096137046814f, -4.698165416717529f, 4.53480863571167f, -2.2949156761169434f, 2.932422399520874f, 18.38007926940918f, 10.144601821899414f, 1.8887478113174438f, 17.05137825012207f, 2.2348875999450684f, 2.445681571960449f, -16.11703109741211f, 3.6988015174865723f, 28.14395523071289f, -9.436356544494629f, -4.76363468170166f, -9.584181785583496f, -0.4704954922199249f, 1.5917537212371826f, -21.702960968017578f, 6.3507513999938965f, -3.284785270690918f, 7.535861968994141f, 11.226126670837402f, 9.685650825500488f, 0.8183719515800476f, -15.07618236541748f, 6.108888149261475f, 16.537776947021484f, 12.913955688476562f, -4.378338813781738f, 7.780423641204834f, 7.1294660568237305f, 1.4495526552200317f, 0.26958322525024414f, 4.329139709472656f, 6.674724102020264f, -6.520862579345703f, -2.903130531311035f, -17.643451690673828f, 7.825291633605957f, -4.506794452667236f, 0.16557633876800537f, 7.2597246170043945f, 8.140376091003418f, -9.9315185546875f, -6.21274471282959f, -3.006944179534912f, 4.115133285522461f, 1.6233476400375366f, -8.518281936645508f, 1.5647332668304443f, -7.484006881713867f, -4.436017036437988f, 9.502815246582031f, -7.330345630645752f, 7.664921760559082f, -4.133516788482666f, 3.0703797340393066f, 7.408110618591309f, -1.3500370979309082f, -3.113905191421509f, -1.6690822839736938f, 9.191621780395508f, 0.6589356064796448f, -7.368348121643066f, 11.585264205932617f, 2.6871490478515625f, 5.345765113830566f, 1.0851669311523438f, -4.77912712097168f, -9.00320816040039f, -8.105009078979492f, 3.760385274887085f, 8.186501502990723f, 13.72744369506836f, 4.232875823974609f, 4.095627307891846f, 1.6133935451507568f, 14.020605087280273f, -17.89886474609375f, 10.849470138549805f, -4.106210708618164f, -1.5198999643325806f, -4.078025817871094f, 3.1078972816467285f, 6.467252254486084f, 8.850523948669434f, -1.770440936088562f, -3.601609706878662f, 12.48472785949707f, 1.1591122150421143f, 3.630988597869873f, 0.35214078426361084f, 9.08293628692627f, -9.61496353149414f, -3.6545019149780273f, 4.853860378265381f, -21.803064346313477f, 0.9578412175178528f, -5.081964492797852f, -3.8409125804901123f, 13.125861167907715f, 8.514156341552734f, 2.2802765369415283f, -1.5544071197509766f, 7.989774227142334f, -0.9534971117973328f, -20.655189514160156f, 5.728123664855957f, -4.578756809234619f, -9.761033058166504f, -2.411184072494507f, -1.53468918800354f, 1.7299505472183228f, 4.49857759475708f, -5.621171951293945f, -15.699580192565918f, -1.6200847625732422f, 0.2511891722679138f, 12.249534606933594f, -6.566256523132324f, -14.719828605651855f, -11.666041374206543f, 1.057192087173462f, -1.8811215162277222f, -4.319576263427734f, 9.619815826416016f, -6.46962833404541f, -3.5396673679351807f, -4.881474494934082f, -0.6955579519271851f, 3.299593687057495f, 11.470381736755371f, 1.12203049659729f, -0.921830952167511f, 5.743063449859619f, 6.646578311920166f, 1.6719251871109009f, -11.525588989257812f, -12.89569091796875f, 7.118905544281006f, -14.61849594116211f, -9.530622482299805f, -6.113729476928711f, 0.5888143181800842f, -17.381607055664062f, 15.451555252075195f, -6.282611846923828f, -12.82925033569336f, 1.7211908102035522f, -12.8479585647583f, 3.2159461975097656f, 2.13104510307312f, 0.8415284752845764f, -4.35218620300293f, -6.643268585205078f, -3.2159981727600098f, -1.1216778755187988f, -9.839357376098633f, -6.70488977432251f, -4.389339447021484f, -3.2769744396209717f, -5.768002510070801f, -8.732684135437012f, -5.264852046966553f, 6.483272552490234f, 9.102012634277344f, -14.003499984741211f, 0.17809408903121948f, 15.125717163085938f, -12.025708198547363f, -6.3086838722229f, 14.6791353225708f, 9.44178581237793f, -8.969319343566895f, -5.2899250984191895f, -9.268863677978516f, -6.997377872467041f, 9.275823593139648f, 1.1657955646514893f, -7.220368385314941f, 10.789793968200684f, 10.304367065429688f, -4.045272350311279f, -6.108043193817139f, -2.014784812927246f, -11.121515274047852f, -18.38974380493164f, -8.347667694091797f, 5.974706649780273f, 10.243739128112793f, 5.663839340209961f, 8.547972679138184f, -7.52782678604126f, 5.363646507263184f, 15.58993148803711f, 7.026348114013672f, 2.677098512649536f, 18.409914016723633f, -10.548135757446289f, 3.4355015754699707f, 1.4359816312789917f, -2.546203374862671f, -6.266178131103516f, 11.344420433044434f, -19.2717227935791f, -6.462072372436523f, 11.376580238342285f, 3.8701956272125244f, -8.504956245422363f, -7.087599277496338f, -11.178908348083496f, 0.2572647035121918f, 4.125010967254639f, 9.298619270324707f, -8.014323234558105f, 12.535018920898438f, 6.390746116638184f, -6.37876558303833f, 6.679197788238525f, -9.323160171508789f, -5.269445419311523f, 2.596299886703491f, 0.11649034917354584f, 17.804271697998047f, 4.594757556915283f, 6.764013767242432f, 0.6217585206031799f, -7.7316412925720215f, 8.208429336547852f, -7.3180928230285645f, -9.159605979919434f, 8.136429786682129f, -15.975767135620117f, -5.516753673553467f, -7.645376205444336f, -6.816018104553223f, -1.813738226890564f, -0.754987359046936f, 10.540684700012207f, 6.572773456573486f, 4.151686191558838f, 5.905804634094238f, -0.29068470001220703f, -5.357999324798584f, -5.847013473510742f, -4.441774845123291f, 1.8924025297164917f, -1.1913574934005737f, 14.646678924560547f, -13.63921070098877f, 1.4032258987426758f, -2.480900526046753f, 0.2637772262096405f, -7.374143600463867f, 5.216763019561768f, -22.21915626525879f, -1.6103098392486572f, 9.66075325012207f, 7.245777606964111f, -0.4308447539806366f, 3.232591152191162f, 7.705619812011719f, -3.5778160095214844f, -4.7608866691589355f, -3.9943084716796875f, -2.1508443355560303f, 7.583977222442627f, -11.541088104248047f, -9.87782096862793f, -0.3558848798274994f, -0.8333123326301575f, 0.36477258801460266f, -6.956092834472656f, -0.47873327136039734f, -7.288418769836426f, 1.8383797407150269f, -6.37021541595459f, -1.270717740058899f, -5.504342555999756f, -7.638356685638428f, 3.2992103099823f, 3.3208279609680176f, -7.895376682281494f, -0.4653431177139282f, -5.259646415710449f, 0.3640040159225464f, -4.201903820037842f, 7.085029125213623f, 0.7169751524925232f, 1.866424798965454f, 2.98149037361145f, 2.421513319015503f, -4.835546493530273f, -3.400444746017456f, -13.251764297485352f, -7.206017971038818f, 5.5685553550720215f, 8.815764427185059f, 1.3367613554000854f, -12.028512001037598f, -4.593379974365234f, -5.485569000244141f, 7.575770378112793f, 6.186659336090088f, 4.837860107421875f, 4.19438362121582f, -2.0015146732330322f, -8.463085174560547f, 4.390081882476807f, -0.967718243598938f, -6.633697032928467f, 16.146509170532227f, -12.346258163452148f, 5.003246307373047f, 5.257402420043945f, 5.420393943786621f, -4.056494235992432f, -22.63052749633789f, -3.4521031379699707f, 0.5595429539680481f, -0.6451472043991089f, 7.716019630432129f, -3.91341233253479f, 3.537018299102783f, 0.5044690370559692f, -6.045408725738525f, -3.7082343101501465f, 8.852910995483398f, -3.696688175201416f, -2.3492298126220703f, -9.58127212524414f, -12.638714790344238f, 1.8595764636993408f, -4.140509128570557f, -7.7006120681762695f, 1.3192996978759766f, -9.62076473236084f, 9.613175392150879f, 16.121721267700195f, 7.089335918426514f, -11.461114883422852f, -9.27320384979248f, 4.465617656707764f, 1.9801151752471924f, 11.973653793334961f, -9.35599136352539f, 5.719399929046631f, -7.321558475494385f, 17.419307708740234f, -10.423422813415527f, 4.022407531738281f, -1.3151211738586426f, -5.239253044128418f, 0.22252266108989716f, -23.80808448791504f, 11.66386890411377f, -15.182354927062988f, 2.0668842792510986f, -3.095346450805664f, -10.253707885742188f, 4.4772186279296875f, -16.292877197265625f, 2.14457106590271f, 7.256747245788574f, 0.5063409805297852f, -1.0078110694885254f, -17.32522201538086f, -2.448883295059204f, -7.290302753448486f, -10.84791374206543f, 4.010873317718506f, 0.7512247562408447f, -5.4202561378479f, 2.328165292739868f, -6.0993428230285645f, -12.868612289428711f, 4.567487716674805f, -10.598518371582031f, -11.058136940002441f, -1.659664273262024f, 10.452244758605957f, 18.823354721069336f, -8.08329963684082f, 4.401593208312988f, 2.138773202896118f, -13.085124969482422f, 2.258606195449829f, -2.7691009044647217f, 5.584531784057617f, 10.423367500305176f, 2.846891164779663f, -4.120229721069336f, 1.5436748266220093f, -13.3086519241333f, -8.132325172424316f, 13.937261581420898f, -14.859661102294922f, -10.103026390075684f, 10.011719703674316f, -0.9183381795883179f, 9.600658416748047f, -9.821544647216797f, -9.450371742248535f, -2.242305278778076f, -6.559144020080566f, -4.224136829376221f, -11.820085525512695f, -6.345705986022949f, 8.94025993347168f, 4.509256839752197f, -2.219336748123169f, -0.8802061080932617f, -14.329898834228516f, 7.088708877563477f, -9.526060104370117f, 8.278675079345703f, 2.4871132373809814f, -9.620868682861328f, 9.28262710571289f, 14.8040132522583f, -19.852964401245117f, 4.130612850189209f, 5.440346717834473f, 7.126719951629639f, 0.05489916726946831f, -1.4817479848861694f, 6.829895973205566f, 12.818050384521484f, 9.886153221130371f, 1.254286766052246f, -8.354349136352539f, -11.958121299743652f, 9.16726016998291f, 4.949143886566162f, -7.31182861328125f, 18.494693756103516f, -9.817893028259277f, -0.3160404562950134f, -7.171751022338867f, 15.233163833618164f, -2.146108388900757f, 5.443892478942871f, 4.517480850219727f, 14.043611526489258f, 10.903755187988281f, -7.276175022125244f, 13.366379737854004f, -8.15054702758789f, 0.6368334293365479f, 9.139420509338379f, -6.253448963165283f, 7.538908958435059f, -4.911654949188232f, 1.2663601636886597f, 5.10722017288208f, -10.726715087890625f, -9.355097770690918f, -0.3182983100414276f, -0.6880562901496887f, 1.4052690267562866f, 8.107300758361816f, -17.92100715637207f, -4.534452438354492f, 0.5386764407157898f, 12.413888931274414f, -7.201480388641357f, -13.179725646972656f, -8.37654972076416f, -0.3154694736003876f, 3.091693878173828f, -1.0480709075927734f, 3.751927614212036f, 5.383627414703369f, 6.276157855987549f, 5.926423072814941f, -12.69626235961914f, 0.419791042804718f, 5.777501583099365f, -2.8398115634918213f, -4.607639789581299f, 6.945610523223877f, -3.02091383934021f, -9.469738006591797f, -7.620004177093506f, -2.214186191558838f, -10.362244606018066f, -3.8059444427490234f, -8.182043075561523f, 12.854665756225586f, -9.528903007507324f, 0.49294862151145935f, -8.579874038696289f, 2.118196725845337f, 7.62219762802124f, -5.029037952423096f, 12.290265083312988f, 0.5394968390464783f, 0.8183786869049072f, 1.3815820217132568f, -8.656240463256836f, -10.229368209838867f, 5.616023540496826f, 1.3402541875839233f, 0.14747254550457f, 5.180647373199463f, -5.916377544403076f, 11.535149574279785f, 6.629460334777832f, -2.8352251052856445f, 2.0122580528259277f, -0.12555190920829773f, 0.09699758142232895f, -3.735365867614746f, 6.413780212402344f, -9.827960968017578f, 4.477472305297852f, 3.9674675464630127f, 13.4703950881958f, -10.233613014221191f, 6.505621433258057f, -0.8457767963409424f, 2.9881680011749268f, -1.2331901788711548f, 0.2708655893802643f, -20.598543167114258f, -3.6357667446136475f, -10.855840682983398f, 1.586439847946167f, 10.175971031188965f, -1.9043898582458496f, 2.3536105155944824f, 4.586009502410889f, -16.701967239379883f, -5.804449558258057f, -2.78130841255188f, -12.884259223937988f, 4.541162967681885f, -2.2364847660064697f, 7.2929511070251465f, -20.930879592895508f, -1.1628355979919434f, -0.6431397795677185f, -4.187624454498291f, 9.130221366882324f, 5.2739481925964355f, -5.0708441734313965f, 0.17360350489616394f, 0.5822219252586365f, -0.22339344024658203f, -2.761117458343506f, -10.778762817382812f, 5.385908603668213f, -7.857998371124268f, 2.606891393661499f, 18.175046920776367f, -2.693230390548706f, 10.186522483825684f, -10.02297306060791f, -2.4003751277923584f, 4.620128631591797f, 11.451003074645996f, -9.571845054626465f, -7.237552165985107f, 16.154325485229492f, -19.152278900146484f, 0.16639988124370575f, -1.1797699928283691f, -3.471789836883545f, 7.653792858123779f, 3.1148579120635986f, -17.236486434936523f, 7.4174604415893555f, 5.9619293212890625f, 3.140089511871338f, -1.4843370914459229f, -11.55695629119873f, 4.42191219329834f, -1.2158066034317017f, 1.4187449216842651f, -4.891499042510986f, -0.7901540398597717f, 3.7802929878234863f, -5.735926151275635f, -6.485677242279053f, -1.5401825904846191f, 0.4863590598106384f, 7.93129301071167f, -1.8886117935180664f, 13.101932525634766f, -9.918781280517578f, 7.965061664581299f, -7.907785892486572f, -1.5377079248428345f, 13.429678916931152f, -9.150714874267578f, 11.128018379211426f, 14.368767738342285f, 3.5130884647369385f, -20.24823570251465f, 5.9857892990112305f, -16.631755828857422f, 6.948392391204834f, -0.7420387864112854f, 13.481123924255371f, -6.017175674438477f, -6.115068435668945f, -3.0654659271240234f, 13.489151000976562f, -2.2497708797454834f, 15.904927253723145f, -5.288370132446289f, -2.09615421295166f, -1.6209585666656494f, -10.90034008026123f, -2.202252149581909f, -7.51241397857666f, 1.12482488155365f, -3.7093164920806885f, 5.637340068817139f, 3.773543357849121f, 1.558279275894165f, 0.7900522947311401f, 11.120495796203613f, -4.0744709968566895f, 10.195045471191406f, -1.018953561782837f, 15.167667388916016f, 2.601929187774658f, 0.008113066665828228f, -9.717181205749512f, 6.3429789543151855f, 21.926603317260742f, -9.98530387878418f, 14.128251075744629f, 6.466258525848389f, -6.853448390960693f, -4.719650745391846f, -11.167947769165039f, -3.2867848873138428f, -12.95023250579834f, -5.806682109832764f, 11.830602645874023f, -5.1555986404418945f, -6.032625198364258f, -7.19350528717041f, -0.5742126703262329f, -20.182292938232422f, -2.5560240745544434f, 8.388191223144531f, 3.965162754058838f, -2.2216339111328125f, -11.404376983642578f, 15.09308910369873f, 8.153555870056152f, -3.3941447734832764f, -5.3842549324035645f, 17.98311996459961f, -0.8321418166160583f, -13.44106388092041f, -20.923404693603516f, -7.57475471496582f, -3.0816946029663086f, 14.614544868469238f, 3.570502519607544f, 4.2990217208862305f, 0.1919950395822525f, 1.0185343027114868f, -8.14962387084961f, 10.906466484069824f, 15.584993362426758f, -0.4128416180610657f, -6.817087173461914f, 17.120927810668945f, 2.2402291297912598f, -1.9622392654418945f, -3.413076162338257f, 6.840425968170166f, -5.948558330535889f, 6.701462745666504f, 0.8215048909187317f, -1.0856260061264038f, -3.9645702838897705f, 1.4618407487869263f, -1.7632997035980225f, 0.5018937587738037f, -2.7499217987060547f, 4.106265068054199f, -12.30443000793457f, 6.145236492156982f, 4.5076003074646f, 3.7207319736480713f, 9.945277214050293f, 1.5612727403640747f, -25.549476623535156f, -4.634391784667969f, -9.542594909667969f, 10.549148559570312f, 2.471867322921753f, 1.1155062913894653f, -2.9077959060668945f, 7.583897590637207f, -2.4130048751831055f, 8.161357879638672f, -6.786379337310791f, 15.053579330444336f, 2.3771886825561523f, 8.431512832641602f, -10.34504508972168f, 3.9834461212158203f, 0.3579562306404114f, -14.247590065002441f, -11.214945793151855f, -6.256213188171387f, 8.011175155639648f, 9.249755859375f, -11.789390563964844f, -14.64880657196045f, -0.27090367674827576f, -0.2744891941547394f, 11.454398155212402f, -14.925914764404297f, -7.099286079406738f, -3.3692843914031982f, 6.4099931716918945f, 6.6391777992248535f, 10.544389724731445f, 3.6568782329559326f, 6.840872764587402f, 12.177023887634277f, 3.4128663539886475f, -3.8373212814331055f, -0.12041311711072922f, -13.457633972167969f, 18.881927490234375f, 8.021130561828613f, -7.102904796600342f, 7.734999179840088f, 3.826378107070923f, -0.7286080718040466f, -6.966921329498291f, -9.949929237365723f, -6.5738348960876465f, -6.720236778259277f, -2.4046998023986816f, -7.772119998931885f, -3.3762123584747314f, 16.03032875061035f, -10.771323204040527f, -0.8427897095680237f, 2.035958766937256f, -9.551191329956055f, 13.150819778442383f, -8.132336616516113f, -0.9750276803970337f, 2.9933242797851562f, -12.041450500488281f, -7.16148042678833f, -7.287316799163818f, 5.334017753601074f, -7.423707962036133f, -7.503839015960693f, -13.0750150680542f, -1.5110211372375488f, -3.3898580074310303f, 6.609547138214111f, -1.1499853134155273f, 6.6649370193481445f, -4.917843341827393f, -8.266522407531738f, 16.943111419677734f, 0.35688164830207825f, 2.3017640113830566f, -2.517993927001953f, -20.720897674560547f, -5.1919379234313965f, 6.862222194671631f, 8.344823837280273f, -3.240790367126465f, -0.7951709032058716f, -7.633636951446533f, 1.9410372972488403f, -3.9540672302246094f, -8.897167205810547f, 9.550298690795898f, -12.049423217773438f, 3.2460708618164062f, -0.7465460896492004f, -7.343610763549805f, -25.89638328552246f, 1.305576205253601f, 0.7940993309020996f, 15.786218643188477f, -5.214616298675537f, 5.427506923675537f, -9.456047058105469f, -4.363546371459961f, 14.75342845916748f, -8.062644958496094f, 12.066773414611816f, -17.939416885375977f, -0.6330687403678894f, -8.203537940979004f, -15.027321815490723f, -7.841004848480225f, -6.411040306091309f, -22.517040252685547f, -4.999447822570801f, -8.532695770263672f, 8.279166221618652f, 7.497697830200195f, 8.378273963928223f, -2.62188458442688f, -3.915342330932617f, 3.5234627723693848f, -6.344028949737549f, 7.4199934005737305f, -4.750202655792236f, 1.74264395236969f, 8.607345581054688f, -10.318155288696289f, -0.5510790944099426f, 24.487709045410156f, -4.434352397918701f, 3.7080373764038086f, -16.31922149658203f, 2.130758285522461f, -1.8111369609832764f, 16.641450881958008f, -15.62347412109375f, 9.540854454040527f, -3.6384425163269043f, 1.0087566375732422f, 2.4287056922912598f, 1.3515430688858032f, -1.9539055824279785f, 7.968528747558594f, 3.1463985443115234f, 4.322893142700195f, -4.588891506195068f, 7.193646430969238f, 3.6794357299804688f, -3.6867868900299072f, -12.593381881713867f, 22.773561477661133f, -2.592869997024536f, -8.03166389465332f, 1.6411826610565186f, -5.852877140045166f, 6.320628643035889f, -1.24896240234375f, 2.633263349533081f, 2.3571205139160156f, 2.6310131549835205f, -4.19133996963501f, 9.353876113891602f, -15.862900733947754f, -11.460186958312988f, -0.3524762690067291f, 3.5821545124053955f, -4.435649871826172f, -1.1791728734970093f, 2.729346990585327f, -1.2911462783813477f, -0.1667870581150055f, 3.152031421661377f, -20.307052612304688f, -16.968524932861328f, -2.394564628601074f, -4.503590106964111f, 3.181205987930298f, 0.05041913315653801f, -0.3948255181312561f, -6.3789544105529785f, 1.732195258140564f, -5.6798787117004395f, 7.7667107582092285f, 9.2898530960083f, -4.316888332366943f, -16.857954025268555f, -5.379256725311279f, 4.286224842071533f, 4.1023759841918945f, -8.238680839538574f, 9.628629684448242f, -2.3275868892669678f, 2.8554039001464844f, -8.514399528503418f, -9.410696029663086f, 4.0246405601501465f, -6.667844295501709f, -5.36508321762085f, 0.8656852841377258f, 2.9765312671661377f, -0.6338213682174683f, -4.245971202850342f, 6.7638702392578125f, 8.446202278137207f, -2.508578300476074f, -0.5704231858253479f, 0.46178925037384033f, 10.725545883178711f, 4.313973426818848f, -6.148879528045654f, 3.38698410987854f, -12.07677173614502f, 8.890965461730957f, 9.622424125671387f, -1.9190295934677124f, 1.035591721534729f, 3.041085720062256f, 11.527141571044922f, 2.2250008583068848f, 0.06786001473665237f, 11.618230819702148f, 0.2951813340187073f, 7.107181549072266f, -13.101400375366211f, -4.043383598327637f, 5.551668643951416f, 0.23836034536361694f, -9.364093780517578f, 0.38196349143981934f, 5.709733486175537f, -3.463144540786743f, 10.42895793914795f, -21.435224533081055f, -22.471851348876953f, 2.9046216011047363f, 5.418637275695801f, -2.009808301925659f, -12.568074226379395f, 5.197458744049072f, 12.429717063903809f, -3.8452060222625732f, -8.332062721252441f, -1.3275021314620972f, 8.634135246276855f, -17.408498764038086f, 4.173489570617676f, 3.602097272872925f, 16.919105529785156f, -14.429887771606445f, -7.9446563720703125f, -15.02901554107666f, 5.9782915115356445f, -2.348747491836548f, 2.622194766998291f, 9.501197814941406f, 6.67450475692749f, 5.712249279022217f, -1.8605291843414307f, -0.874760627746582f, 15.388128280639648f, 2.725569486618042f, -4.520296096801758f, -11.915221214294434f, 3.2612195014953613f, -10.38715934753418f, 2.1763875484466553f, -1.2297403812408447f, 6.042415618896484f, -13.808868408203125f, -5.520578861236572f, -1.4444622993469238f, -7.977278232574463f, 8.31882381439209f, -9.937228202819824f, 7.792935848236084f, -8.416111946105957f, 7.689346790313721f, -4.904921054840088f, -0.9467958211898804f, -2.2417097091674805f, 4.855114459991455f, 12.061532020568848f, -4.05377721786499f, 7.5981645584106445f, -5.726215362548828f, -5.700634956359863f, -3.6235997676849365f, -8.09415340423584f, -8.389449119567871f, 3.4986400604248047f, -0.17018863558769226f, -4.8295674324035645f, 3.72519588470459f, 6.054676532745361f, 2.607694149017334f, -5.292807579040527f, 10.3839111328125f, 2.9594953060150146f, 4.974825382232666f, 30.000545501708984f, -6.096185207366943f, 0.5663076043128967f, 4.246780872344971f, -7.833420753479004f, -6.573525905609131f, -4.979223251342773f, 9.320971488952637f, 2.075157642364502f, 0.24154403805732727f, 13.304895401000977f, -9.484021186828613f, 8.853174209594727f, -7.756874084472656f, 0.7791877388954163f, 13.578595161437988f, -4.487305641174316f, -1.5067740678787231f, -20.156299591064453f, 0.3791326582431793f, -4.492989540100098f, 11.668903350830078f, -3.7837438583374023f, -0.9617695808410645f, -15.763601303100586f, 12.063730239868164f, 0.8532925248146057f, -9.071452140808105f, 3.879969358444214f, 14.453145027160645f, -2.9419867992401123f, 8.816530227661133f, 6.016716957092285f, -15.384603500366211f, 7.9487481117248535f, 5.712857723236084f, 6.595734596252441f, -8.348841667175293f, -7.271892070770264f, 8.39649772644043f, 3.010307550430298f, -7.354966640472412f, 4.326541900634766f, 6.269808292388916f, -2.21332049369812f, 2.8673322200775146f, -11.210542678833008f, 8.31998062133789f, -12.9059419631958f, 8.617777824401855f, -3.5901377201080322f, -10.140533447265625f, 0.8802559971809387f, -9.837919235229492f, -9.109603881835938f, -15.81827449798584f, 4.78360652923584f, -1.1366561651229858f, -22.160612106323242f, -12.40213680267334f, -18.063514709472656f, -2.445664882659912f, 15.82851505279541f, -4.466148853302002f, 7.4260993003845215f, 10.562188148498535f, 2.9546096324920654f, 6.616511821746826f, -11.446455001831055f, 1.037653923034668f, 8.754400253295898f, 9.139323234558105f, -6.899957180023193f, -8.12019157409668f, 10.253373146057129f, 3.2580606937408447f, -1.1452258825302124f, 6.390077114105225f, -1.9534237384796143f, -3.5342469215393066f, -6.437942981719971f, -11.188612937927246f, 1.4919053316116333f, 12.896910667419434f, 4.564462184906006f, -8.683686256408691f, 2.0561792850494385f, -7.927750587463379f, -7.966180324554443f, 3.0324785709381104f, -14.164445877075195f, -0.5540791153907776f, 7.670888900756836f, -0.11896628886461258f, 1.1451127529144287f, 22.234376907348633f, -5.0322980880737305f, 10.48377513885498f, 10.193860054016113f, -6.44713020324707f, 4.20560359954834f, -19.378570556640625f, 7.226066589355469f, -11.02578067779541f, 16.957338333129883f, 5.088015079498291f, 7.4178314208984375f, 0.37720149755477905f, 17.275354385375977f, 5.082028865814209f, 0.11316994577646255f, 10.450178146362305f}; - -// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel). -// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad -// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs. -// quad ptr for (m_tile mt, k_group g) = gemm_Apack_dram + (mt*(K/4)+g)*64. -static float gemm_Apack_dram[64*64] __attribute__((section(".data"))) = -{1.9269152879714966f, 1.4451338052749634f, 1.931160569190979f, 1.7743884325027466f, -0.685481607913971f, 0.4334380030632019f, 0.6663737893104553f, -1.3838845491409302f, -1.363769769668579f, -0.8032152652740479f, -1.2786966562271118f, 0.3220294117927551f, 0.34436315298080444f, -0.22056575119495392f, -0.7111086845397949f, -0.447700172662735f, 1.4872840642929077f, 0.8564125299453735f, 1.0118638277053833f, -0.921550989151001f, 0.5635589957237244f, -0.7171905636787415f, -0.07426456362009048f, 0.4868715703487396f, 0.1929578185081482f, -1.1209005117416382f, -0.03842746838927269f, 0.44605663418769836f, -3.1016058921813965f, 0.7118127346038818f, -0.3866667151451111f, -0.7288169264793396f, 0.9007171988487244f, 2.218075752258301f, -1.4364064931869507f, 0.9624499082565308f, -1.507175087928772f, 1.055369257926941f, -0.20960482954978943f, -1.0020296573638916f, -0.6103342771530151f, 0.1956406533718109f, 1.9138009548187256f, 1.5229592323303223f, -1.4587225914001465f, 0.3415871560573578f, 0.9578314423561096f, -0.16066236793994904f, -2.1055209636688232f, 0.5231655240058899f, -1.1298598051071167f, -0.33701515197753906f, -1.610666036605835f, -1.4533969163894653f, 0.16632132232189178f, 0.0329488180577755f, 0.16323445737361908f, -0.7815181016921997f, 0.3378446102142334f, 1.2804899215698242f, -1.4318257570266724f, 1.5885895490646362f, -0.8225308656692505f, -0.32063713669776917f, 0.6784184575080872f, 0.34664666652679443f, -0.1360345333814621f, -1.1753336191177368f, -1.4790465831756592f, 0.46515071392059326f, 1.4703037738800049f, -0.42919591069221497f, 1.51017165184021f, -1.7898789644241333f, 0.12505611777305603f, -0.11616043001413345f, -0.6071268916130066f, -0.3488781750202179f, -2.3908050060272217f, -0.6307737827301025f, -1.2345448732376099f, -0.19733144342899323f, 1.6354095935821533f, 0.35805708169937134f, 0.4322742819786072f, 0.37139150500297546f, -0.9390866756439209f, -0.9817978739738464f, 0.21230429410934448f, -0.26157355308532715f, -0.7621514797210693f, 1.3705363273620605f, -0.25973787903785706f, -0.45791950821876526f, 0.322247713804245f, -0.788766622543335f, -0.04306747764348984f, -1.0545889139175415f, 0.6547407507896423f, 0.47876790165901184f, -0.1250254064798355f, -0.004656785633414984f, -0.6013189554214478f, -0.6420586109161377f, -0.7252011299133301f, -0.44025033712387085f, -1.190559983253479f, -0.4809381365776062f, -0.7190185785293579f, -1.2322070598602295f, 1.875388741493225f, 1.3061575889587402f, -1.6046669483184814f, 1.2779955863952637f, 0.5760045647621155f, 1.353700041770935f, 0.7821183800697327f, 0.07954943925142288f, -0.09964022785425186f, 0.8265887498855591f, -0.9527732729911804f, 2.1848294734954834f, 0.7756073474884033f, -0.9903622269630432f, -0.38583096861839294f, -0.598077118396759f, 1.1042989492416382f, -0.9275763630867004f, -0.7521352767944336f, -0.1721901297569275f, 1.1415079832077026f, 0.5260620713233948f, -1.598767638206482f, 0.3781784772872925f, -0.9851518273353577f, 1.591395616531372f, 0.5216943025588989f, -0.48009708523750305f, 0.455719918012619f, -1.3641812801361084f, 0.5233525037765503f, -0.28154700994491577f, -0.5223758816719055f, -0.26273947954177856f, 1.6487230062484741f, 0.5237884521484375f, 0.018564576283097267f, 2.1120378971099854f, -0.10912995040416718f, 0.7051141262054443f, -2.488459348678589f, -0.1208132952451706f, -0.46386733651161194f, -1.2871733903884888f, 0.2503303289413452f, 0.008205652236938477f, -0.8211768269538879f, 0.05281926319003105f, -0.7401803731918335f, 0.9314952492713928f, -0.3924786448478699f, 0.056621819734573364f, -1.8058050870895386f, -0.5207571387290955f, 0.7151994705200195f, -1.7236974239349365f, -0.33131900429725647f, -0.48302069306373596f, 0.18237744271755219f, 0.7388824224472046f, -1.3610970973968506f, -0.40586018562316895f, -0.47086891531944275f, 0.42497751116752625f, 0.16235657036304474f, -0.4593467116355896f, -1.4036071300506592f, 0.4262961447238922f, 0.9254348874092102f, -0.9320061206817627f, 0.03913922235369682f, -0.8434810638427734f, 0.8435799479484558f, 0.11329790204763412f, -0.38666075468063354f, 0.03389474004507065f, 1.8018341064453125f, -0.7110859751701355f, 0.6016423106193542f, 0.4825834333896637f, -0.2369976043701172f, -0.9419456720352173f, -0.7278813123703003f, 0.575005054473877f, -0.3753443658351898f, 0.18516133725643158f, 1.305860161781311f, 0.4351435601711273f, 0.9874473810195923f, 0.0771508663892746f, -1.7906768321990967f, -0.31229403614997864f, -0.07434194535017014f, -0.3495793640613556f, -0.28251126408576965f, 0.48813387751579285f, 0.5099347233772278f, -0.7089186310768127f, -0.5594301819801331f, -0.6417241096496582f, 1.0330873727798462f, 1.0686918497085571f, 0.24659274518489838f, 0.26588720083236694f, -0.33197471499443054f, -0.9228128790855408f, 0.09329313784837723f, -0.2541753351688385f, -0.15664155781269073f, 0.3797488212585449f, 0.7692679762840271f, 1.0082393884658813f, 1.670624852180481f, 2.1860759258270264f, -0.7688388824462891f, -2.2063984870910645f, -0.6866509318351746f, 1.3065344095230103f, -1.9775909185409546f, -0.5870985388755798f, -0.8076189756393433f, -1.2619991302490234f, -1.9152568578720093f, -1.205536127090454f, -0.8708454966545105f, 0.9993040561676025f, -0.7668924331665039f, -0.595004141330719f, 1.5921050310134888f, -0.6493158936500549f, 0.7624453902244568f, -0.7508030533790588f, 0.6368136405944824f, 0.4598345160484314f, 0.01789604313671589f, 0.0826888456940651f, 0.824364185333252f, 1.0860532522201538f, -0.6421752572059631f, -0.9542103409767151f, -0.6410972476005554f, 1.2751853466033936f, -0.9494866728782654f, 0.3926331400871277f, -0.41619211435317993f, 0.45214036107063293f, 1.6423169374465942f, 0.01086814422160387f, -0.9726738929748535f, -0.8146268725395203f, -1.3793021440505981f, 0.8853808045387268f, 0.024699924513697624f, 1.096641182899475f, 1.3438509702682495f, 0.061276569962501526f, -0.414562851190567f, 0.9594927430152893f, 0.01691739819943905f, 0.8229668736457825f, 1.861944556236267f, 0.8520749807357788f, -0.1595974713563919f, -0.33874234557151794f, 0.9584577679634094f, -1.0212392807006836f, 0.625802755355835f, 0.1824439913034439f, -1.0641486644744873f, -0.6836934685707092f, -1.2922308444976807f, 0.08526104688644409f, -0.6902380585670471f, 0.10350999981164932f, 0.08027740567922592f, -0.886031985282898f, -1.077892780303955f, -1.6946725845336914f, -0.4973975419998169f, -1.3406795263290405f, 1.6192004680633545f, -0.49492356181144714f, -2.5849502086639404f, 0.7863810062408447f, -0.7601934671401978f, 0.06604336202144623f, 0.766244113445282f, 0.7481252551078796f, -0.22995619475841522f, 0.8290349841117859f, 0.7448412775993347f, 1.4801039695739746f, 0.8848565220832825f, 1.1805996894836426f, 0.439589262008667f, -0.5853705406188965f, 1.450609803199768f, -0.5922516584396362f, -0.02399955503642559f, -0.057920295745134354f, -0.4075061082839966f, -0.0007737990817986429f, 0.64540034532547f, -0.16356196999549866f, -2.172283887863159f, 2.0921294689178467f, 1.345484972000122f, 0.8391514420509338f, -0.8342104554176331f, -2.8929238319396973f, -0.7581311464309692f, 0.5361881256103516f, 0.2694815397262573f, 0.15431594848632812f, -0.1221928745508194f, 0.5666652917861938f, 0.9623646140098572f, 0.1620604395866394f, 0.353316068649292f, -0.9085567593574524f, 0.08768323808908463f, 0.7953095436096191f, 0.12682189047336578f, -0.20004984736442566f, 1.0300744771957397f, -0.3875778615474701f, 1.078317642211914f, 0.5246226191520691f, -0.21037597954273224f, 0.4407670795917511f, -0.7469954490661621f, -0.7097623348236084f, -0.14264194667339325f, 1.195958137512207f, -2.6474881172180176f, 0.3129958212375641f, 1.0937845706939697f, 0.2792847752571106f, -2.4520716667175293f, 0.9949536919593811f, -0.8680985569953918f, -0.7124031782150269f, 0.8008005619049072f, 1.1412016153335571f, -0.7328027486801147f, -0.14829230308532715f, 1.7093087434768677f, -0.4875054359436035f, 0.15271379053592682f, -1.3061535358428955f, -1.4575366973876953f, 0.8050477504730225f, -0.1177205815911293f, 0.1864478439092636f, 0.4159761369228363f, 0.7201864719390869f, -0.5701602697372437f, -1.6171332597732544f, 1.680620551109314f, 0.05164359509944916f, 0.10429783165454865f, -2.3184432983398438f, 0.05792269483208656f, 0.050095997750759125f, -0.0388023778796196f, -1.4039719104766846f, -0.9712379574775696f, -1.1133604049682617f, -0.29864323139190674f, 0.3547132909297943f, 1.9025356769561768f, -0.13413065671920776f, 0.32332202792167664f, -0.35899198055267334f, 1.27912437915802f, 0.7439519762992859f, 0.3487516939640045f, -0.39799532294273376f, 1.1929805278778076f, 0.6084084510803223f, 0.9446058869361877f, -1.0597201585769653f, 0.25403109192848206f, 0.4981626570224762f, -0.9536206126213074f, 0.09063850343227386f, -0.7346699833869934f, -1.4067999124526978f, 1.1284750699996948f, 0.051366694271564484f, 1.2964228391647339f, -0.4815843999385834f, 0.9675941467285156f, 1.0804862976074219f, 1.9372931718826294f, 1.6308681964874268f, -1.5824053287506104f, 0.3057299852371216f, -0.1790592074394226f, -1.1999552249908447f, -0.09247277677059174f, 1.7422553300857544f, 0.044657133519649506f, -2.3609628677368164f, -1.2123126983642578f, 0.6950237154960632f, 0.610466480255127f, -1.0494661331176758f, -0.46568843722343445f, -1.7808643579483032f, 0.7287133932113647f, -0.08472306281328201f, 0.9871290922164917f, 0.4150581359863281f, 1.1992844343185425f, 0.12711313366889954f, -1.01665461063385f, -1.2660012245178223f, -1.5211198329925537f, -0.2904934287071228f, 2.602391004562378f, 1.835181474685669f, 1.334737777709961f, 0.603898823261261f, 1.6047972440719604f, 1.5080454349517822f, 0.9808937907218933f, 1.0844124555587769f, 1.1456739902496338f, -0.7174144983291626f, -0.4292171895503998f, 0.4403660297393799f, -0.007675689645111561f, 0.38916081190109253f, 0.3478375971317291f, -0.13345853984355927f, -0.09572362899780273f, -1.9180361032485962f, -0.2316243201494217f, -1.7222950458526611f, -2.4801201820373535f, 0.30942854285240173f, 0.41459226608276367f, 0.9477656483650208f, -0.14181147515773773f, 2.833967924118042f, 1.010284185409546f, 0.6377718448638916f, -0.518220841884613f, 0.34287506341934204f, 0.7401772737503052f, -0.15693345665931702f, -0.08114803582429886f, -1.3923954963684082f, 0.041759490966796875f, -0.827768862247467f, -0.4175437390804291f, -0.5003090500831604f, 1.15656316280365f, -0.676629364490509f, -0.2763414680957794f, 1.9534740447998047f, 0.6110401153564453f, 0.15978877246379852f, 0.83954256772995f, -1.4590637683868408f, 1.4161995649337769f, 1.138344645500183f, 1.2586976289749146f, 0.540465772151947f, -0.2515752911567688f, 1.334702968597412f, -1.1954537630081177f, 1.0350031852722168f, 0.2690545618534088f, -0.5730168223381042f, -0.19321373105049133f, 2.0486814975738525f, 1.2208385467529297f, 1.7697970867156982f, 0.05852266773581505f, -1.4936561584472656f, 0.6833979487419128f, -0.2505214214324951f, 0.8691263794898987f, 0.4350730776786804f, 0.859858512878418f, 0.48353928327560425f, 0.8123369216918945f, 1.6896470785140991f, -0.036629438400268555f, -0.3303174376487732f, 0.7767809629440308f, -1.0880382061004639f, -0.6076440215110779f, 0.6268176436424255f, -1.6682480573654175f, -0.22138521075248718f, -0.13825182616710663f, 1.6704555749893188f, -0.9609367251396179f, -2.2717032432556152f, -1.3846737146377563f, -2.5095443725585938f, -1.9005532264709473f, -0.004505051765590906f, 0.9732939004898071f, -0.7939430475234985f, 0.6838752627372742f, 1.621694564819336f, -1.7376028299331665f, -1.8736529350280762f, 2.129624843597412f, 0.22523505985736847f, 0.9212995171546936f, -0.545271098613739f, 0.05182264745235443f, -0.13386189937591553f, -0.8712361454963684f, 0.4880010485649109f, 0.22857652604579926f, 1.666792392730713f, -1.0150787830352783f, 0.3752319812774658f, -1.3245893716812134f, 0.8512656688690186f, -0.12535162270069122f, 2.3259060382843018f, -1.5181471109390259f, -0.07724537700414658f, 0.5282443761825562f, -2.15816330909729f, -0.3284812867641449f, -0.058557309210300446f, -0.223365917801857f, 0.7845868468284607f, 0.02485940419137478f, 0.15392017364501953f, -0.5419175624847412f, 0.08790969103574753f, -0.5160817503929138f, -0.40046969056129456f, -1.3658148050308228f, -0.9203909635543823f, 0.1387282907962799f, 0.9856945276260376f, -0.008228386752307415f, -1.6607975959777832f, -2.247206211090088f, 0.12574470043182373f, 1.7173614501953125f, 0.02864718623459339f, -0.34595024585723877f, -1.0602530241012573f, -0.44102486968040466f, -1.241483449935913f, 0.6001842617988586f, -0.6088271737098694f, 1.111746072769165f, 0.6661149263381958f, -1.1797568798065186f, 1.2783364057540894f, -1.4493319988250732f, -0.6637441515922546f, -0.4478967487812042f, -0.5525766611099243f, 0.3188803195953369f, 0.640755295753479f, 0.2868328094482422f, -0.572657585144043f, -0.3136177957057953f, -0.32025346159935f, -0.4702208340167999f, -0.508095383644104f, -0.6227966547012329f, -0.44026491045951843f, -0.5297411680221558f, 0.28815189003944397f, -0.605182409286499f, 0.3657907545566559f, 0.4234687089920044f, 0.07448001205921173f, -0.42451897263526917f, 0.5832474231719971f, -0.7308424115180969f, 0.0835680365562439f, -0.12925422191619873f, -0.844377875328064f, -0.6086435317993164f, -0.6184902191162109f, -0.7891808748245239f, -2.3179564476013184f, 0.9625157713890076f, 0.869049608707428f, -0.17924511432647705f, -0.39920157194137573f, -0.3874586224555969f, -0.1492866724729538f, 0.3057209253311157f, 1.0669267177581787f, 0.17482025921344757f, 0.39990535378456116f, -0.7149624228477478f, -0.5513465404510498f, -0.046192023903131485f, -1.647040605545044f, -0.167823925614357f, 1.294582724571228f, 0.2794382870197296f, -0.8097057938575745f, 0.19955870509147644f, 0.49674081802368164f, -0.22963792085647583f, -0.5522539615631104f, -0.7745925188064575f, -0.4501533806324005f, -1.0939292907714844f, 1.989207148551941f, -0.047562163323163986f, 1.9889612197875977f, -1.6457397937774658f, -1.0362098217010498f, 1.6433145999908447f, 0.22267311811447144f, -0.5718191266059875f, -1.4298604726791382f, -1.2461947202682495f, -2.369169235229492f, -0.40709349513053894f, -0.09342008084058762f, -1.5575724840164185f, -0.18526747822761536f, -1.6021603345870972f, -0.07198750972747803f, 2.0207436084747314f, 1.900311827659607f, -0.4833274185657501f, -0.4503057301044464f, 2.0070879459381104f, -0.8483412265777588f, -2.7936289310455322f, 0.45901596546173096f, -0.41459938883781433f, -0.5614708065986633f, 0.8702965974807739f, -1.0284309387207031f, 0.9956361055374146f, 0.7527588605880737f, 1.3528969287872314f, -0.906094491481781f, 0.25391900539398193f, 1.6950805187225342f, -0.740294337272644f, -0.0729660615324974f, -1.2531019449234009f, 1.6489421129226685f, -0.7111545205116272f, 0.5309328436851501f, 1.4558700323104858f, -0.5949130654335022f, -1.0552809238433838f, 0.40444278717041016f, -0.8797858357429504f, 0.4047577977180481f, 1.288827657699585f, -2.0487122535705566f, 0.9364385008811951f, 0.028089528903365135f, 0.31428107619285583f, -0.5479549169540405f, 1.118869423866272f, 1.6005686521530151f, 0.5235219597816467f, -1.3614802360534668f, 0.3316534161567688f, 1.2687278985977173f, -1.3284013271331787f, 2.1425962448120117f, -0.6011420488357544f, 0.17846599221229553f, 0.05229547247290611f, -1.0810555219650269f, 0.7122363448143005f, -0.17536965012550354f, 0.1415553092956543f, -1.1425532102584839f, 1.7732776403427124f, -0.07858924567699432f, -1.71055006980896f, 1.9562491178512573f, -1.00010085105896f, 1.2904434204101562f, 0.7060741186141968f, -0.5153723955154419f, -1.2741512060165405f, 0.2649095058441162f, -1.5468504428863525f, 0.01762307994067669f, -0.031765542924404144f, -1.7734959125518799f, 1.0348176956176758f, -0.44875210523605347f, -2.071660280227661f, 0.43104586005210876f, 0.8384853601455688f, 1.7684898376464844f, -0.6919524669647217f, -1.1755682229995728f, 0.35730111598968506f, 1.0827196836471558f, 2.1227850914001465f, 1.2731683254241943f, 0.7567060589790344f, 0.0782259851694107f, 0.10164086520671844f, -0.7046414017677307f, -0.626437783241272f, -0.03045438416302204f, -0.4125255346298218f, 0.3683530390262604f, -0.2698453664779663f, -0.9857985377311707f, -0.47199076414108276f, -0.0783226415514946f, 0.5892837643623352f, 1.2498642206192017f, -1.234653115272522f, -0.0013108636485412717f, 0.7755194902420044f, 0.19315829873085022f, 1.3433040380477905f, -0.39465200901031494f, -0.5150922536849976f, 0.3830311596393585f, -0.9769555926322937f, 0.7637977004051208f, 0.12306158244609833f, -1.2370758056640625f, -1.2894343137741089f, -0.9705761075019836f, 0.9187757968902588f, 0.9821351766586304f, -0.4879138767719269f, -0.30360376834869385f, 2.0265355110168457f, 0.40967342257499695f, 0.7132695913314819f, 1.8868111371994019f, 0.6902899742126465f, -0.04476971551775932f, -0.03363388776779175f, 1.1792222261428833f, 0.8757511377334595f, -2.301875114440918f, 1.0762810707092285f, 1.4723693132400513f, 0.6662830114364624f, 0.22690092027187347f, -0.9138230085372925f, -1.457029104232788f, 0.03581761196255684f, -0.9291303157806396f, 0.4038029611110687f, -0.21844321489334106f, -0.4939993619918823f, 1.179942011833191f, 1.8594977855682373f, -0.4137862026691437f, 0.15132997930049896f, -0.0010087001137435436f, -1.0667427778244019f, 1.4108561277389526f, 0.24650610983371735f, 0.4927920699119568f, -0.6581372618675232f, -0.10233523696660995f, 0.12058872729539871f, 0.2761908769607544f, -0.7139783501625061f, 0.16629981994628906f, 1.1366126537322998f, -0.33142781257629395f, 2.6221468448638916f, 0.5184088349342346f, 0.739393413066864f, -0.8494256734848022f, -1.9893426895141602f, -1.3143675327301025f, 0.1328691989183426f, -0.5128253102302551f, 0.07802387326955795f, -0.5991530418395996f, -0.8056637048721313f, -0.5388752818107605f, 0.8337291479110718f, 2.1441681385040283f, -0.46184006333351135f, 0.6495042443275452f, 0.36905255913734436f, -0.7015367746353149f, 0.27310314774513245f, -1.6593921184539795f, 0.29731303453445435f, -1.31621515750885f, 0.12191462516784668f, 0.3006223440170288f, 0.5258087515830994f, 0.4770564138889313f, -0.20757682621479034f, 0.4625823199748993f, -0.9585452079772949f, 1.7045671939849854f, 1.419979453086853f, 0.09495851397514343f, 0.3802972435951233f, -0.4323408901691437f, 2.7312309741973877f, 0.3062905967235565f, 0.4344584047794342f, -1.2524477243423462f, 0.47808775305747986f, 0.07734657824039459f, -0.48799172043800354f, 0.7261772155761719f, -0.9319478273391724f, -0.8718891143798828f, 0.45363426208496094f, 0.3459012508392334f, 0.848518967628479f, -0.7525874376296997f, 0.19898031651973724f, 0.1414770483970642f, 0.43200522661209106f, 1.182044506072998f, 0.0033933203667402267f, -1.5844100713729858f, 0.2761341631412506f, 0.6477669477462769f, 1.1913690567016602f, 0.09115186333656311f, -1.5909662246704102f, -0.027118360623717308f, 1.2460919618606567f, 0.6424751281738281f, -0.047891248017549515f, -0.647229790687561f, -0.23609064519405365f, 0.07110362499952316f, -0.30918216705322266f, 0.32602694630622864f, -1.0240145921707153f, -2.5446670055389404f, -0.5895728468894958f, -0.4324244260787964f, -0.8140076398849487f, -0.3890652060508728f, -1.13597571849823f, -0.3532457649707794f, -2.3065085411071777f, -0.20395424962043762f, 0.6685602068901062f, -1.2822614908218384f, 0.30340856313705444f, 0.5633530616760254f, -0.09658124297857285f, -0.3894469738006592f, 0.22404761612415314f, 1.3719074726104736f, 0.569182813167572f, 1.1740480661392212f, -0.7359927892684937f, 0.5279164910316467f, -0.52259761095047f, 1.4638569355010986f, -1.2868918180465698f, 0.6853673458099365f, 1.0429801940917969f, 1.96529221534729f, -0.45007675886154175f, -0.5786357522010803f, 1.541925072669983f, 2.8543806076049805f, -0.7554785013198853f, -0.5379461050033569f, -0.7911050319671631f, 0.7011352777481079f, -1.4032478332519531f, -0.012685478664934635f, -0.5187733173370361f, 1.255434274673462f, 0.17988650500774384f, -0.13968797028064728f, 0.6899017095565796f, -0.9638485312461853f, 0.47390419244766235f, -1.083811640739441f, -0.108744777739048f, 0.8243650794029236f, 1.3675810098648071f, 0.7378400564193726f, -0.19896702468395233f, 0.6674330234527588f, 0.03600366786122322f, 0.24083632230758667f, -1.5012763738632202f, -0.7149558067321777f, -2.126762628555298f, -1.1807502508163452f, -1.3129348754882812f, -2.5667941570281982f, 0.6503364443778992f, -0.3889259994029999f, -0.4189043343067169f, 0.7983470559120178f, -0.3197358250617981f, -0.8505349159240723f, -1.3615714311599731f, -0.8035953640937805f, -0.06347727030515671f, 0.13253536820411682f, -1.9266542196273804f, 0.8539193272590637f, -0.13408313691616058f, -1.282929539680481f, 0.03780364990234375f, 0.7096128463745117f, 1.1662380695343018f, 0.8126105666160583f, 1.4384385347366333f, 1.8890222311019897f, -0.9130924344062805f, 0.03610055148601532f, -0.5193602442741394f, -1.3776048421859741f, 0.6756148934364319f, 0.7642406225204468f, 0.1278512328863144f, 0.512991189956665f, -1.0407685041427612f, 0.448485791683197f, -1.1702114343643188f, 0.8198426961898804f, 0.01693599671125412f, 1.4981187582015991f, -0.7068426609039307f, 0.5934628248214722f, 1.919209361076355f, 1.3406710624694824f, 0.07648162543773651f, -0.4410470724105835f, -0.0978068932890892f, 1.095009684562683f, 1.0229133367538452f, 0.5397310256958008f, -0.7647228837013245f, -0.590737521648407f, -0.10318559408187866f, 0.6214459538459778f, 0.5325868129730225f, 0.043896086513996124f, -1.2519514560699463f, 0.0696544423699379f, -1.6514869928359985f, 0.9199973940849304f, 0.34005025029182434f, 0.1417587399482727f, 1.8445940017700195f, 0.3398909568786621f, -0.5557951331138611f, 0.5655050277709961f, -0.05528254434466362f, 0.8540631532669067f, 1.1894739866256714f, 0.42318588495254517f, -0.6035352349281311f, 1.4443233013153076f, 3.0250484943389893f, -1.6034338474273682f, 2.1477253437042236f, -0.3787555396556854f, 1.4557304382324219f, 1.1084681749343872f, -1.184537410736084f, 0.7199674844741821f, 0.7042727470397949f, 0.5057917237281799f, 1.204850673675537f, -0.4900680184364319f, 0.7606944441795349f, -0.33889833092689514f, -0.1742597371339798f, 0.23202891647815704f, 1.3462589979171753f, -0.42982181906700134f, -0.6604134440422058f, -1.5597758293151855f, -0.3461014926433563f, 0.5544233322143555f, 1.3835493326187134f, 0.41140761971473694f, 0.7098760008811951f, 0.22245365381240845f, -0.982473611831665f, -0.35945725440979004f, -0.7463049292564392f, 0.5179733633995056f, 0.6092063784599304f, 0.5064983367919922f, 0.8556069731712341f, 0.5761587619781494f, 0.11352583765983582f, -0.8009540438652039f, -0.2633814215660095f, 1.5817502737045288f, -1.2247875928878784f, 0.23484112322330475f, -1.407819390296936f, -0.09603477269411087f, -0.294853538274765f, -0.5165784358978271f, 0.2704370319843292f, -1.0562596321105957f, -0.8496796488761902f, -0.2919580638408661f, 0.10698884725570679f, -0.9970720410346985f, 0.05950453504920006f, 0.6107109189033508f, 0.5484238862991333f, 0.7350953221321106f, 0.962885856628418f, 0.08861476927995682f, -0.0801108106970787f, 1.0644340515136719f, -0.2798626720905304f, -0.6881742477416992f, -1.1818499565124512f, 0.24130821228027344f, -0.6987038254737854f, 0.45643651485443115f, -0.10149910300970078f, -0.34807199239730835f, 1.7285562753677368f, 0.2142704576253891f, 0.45765289664268494f, -0.269319087266922f, -1.5785412788391113f, -0.3476867079734802f, 0.5194124579429626f, 1.4888246059417725f, 1.0837124586105347f, 0.4755038022994995f, -1.0340323448181152f, 0.18275369703769684f, -0.20516234636306763f, -0.3147018849849701f, 0.08301469683647156f, 0.2176828533411026f, 0.2920781075954437f, -0.17470814287662506f, 0.9676891565322876f, 0.45085909962654114f, 0.6715953946113586f, 0.8490674495697021f, 1.170888900756836f, 0.8825610876083374f, 0.17298388481140137f, -1.4316335916519165f, -0.4947497546672821f, 0.6246524453163147f, -0.7811664342880249f, -0.41328999400138855f, 0.712009072303772f, 1.1277714967727661f, -0.6925867199897766f, -1.7561272382736206f, -0.6673583388328552f, 0.04091703146696091f, -0.060151856392621994f, 0.2014705091714859f, 2.1779797077178955f, -0.23840203881263733f, 0.5123522281646729f, 0.1427735835313797f, -0.8951197266578674f, -0.7939775586128235f, 0.6872723698616028f, 0.3946205973625183f, -0.9005926847457886f, -1.5005147457122803f, -0.8442767858505249f, 1.4259073734283447f, 1.6183183193206787f, -0.06933283060789108f, 0.06978437304496765f, 0.3839779198169708f, 1.7791979312896729f, 0.5468734502792358f, -0.9818529486656189f, 0.6328914165496826f, 0.5340191125869751f, -0.674835205078125f, 0.7835897207260132f, 1.1304746866226196f, 0.8906894326210022f, -0.2404831200838089f, -0.32920387387275696f, 0.5127183794975281f, -0.26440465450286865f, 0.2585761547088623f, -1.6634936332702637f, 1.2309634685516357f, 0.25832492113113403f, -0.06058019772171974f, 1.1258721351623535f, -1.048923373222351f, -0.88661128282547f, -0.3876877427101135f, -1.1108732223510742f, 0.8258382678031921f, 0.47655177116394043f, -0.4854878783226013f, -0.1140289157629013f, -0.4026731848716736f, 1.354137897491455f, 0.4775455594062805f, -0.7650561332702637f, 1.2286686897277832f, -2.4340736865997314f, -0.5304896831512451f, 0.25538599491119385f, -0.5224623084068298f, 0.7963698506355286f, 0.44965043663978577f, -3.106328010559082f, 0.9458276629447937f, -0.8396266102790833f, -0.06616523861885071f, -0.8452204465866089f, 1.9770677089691162f, -0.07092823088169098f, -1.4889180660247803f, 1.2306435108184814f, 0.7042104005813599f, -0.3497500419616699f, -2.0363707542419434f, -0.4588965177536011f, -1.1338030099868774f, 0.6588250994682312f, 0.3726101517677307f, -0.9897713661193848f, -0.15447334945201874f, 0.33320167660713196f, -0.9029294848442078f, 0.3004419207572937f, 0.0267170500010252f, -0.3697128891944885f, -1.2930028438568115f, 0.4252126216888428f, -0.05628490820527077f, -1.338055968284607f, 0.5246880650520325f, -0.9283785820007324f, -0.14128278195858002f, -0.2966694235801697f, -1.9104946851730347f, -0.6022037863731384f, -1.6013476848602295f, -1.2525506019592285f, 0.6440190076828003f, 1.6395184993743896f, -0.25020739436149597f, -0.24597389996051788f, 0.8795658946037292f, -0.016383398324251175f, -1.4897207021713257f, -0.4389103353023529f, -0.6970252990722656f, -0.17175325751304626f, -0.6456266641616821f, 0.23320063948631287f, 0.26085028052330017f, -0.7153372168540955f, -0.059470854699611664f, -0.5745509266853333f, 0.7591805458068848f, -1.0744175910949707f, -0.09136287868022919f, 0.3293827772140503f, 0.07306429743766785f, -0.10749480873346329f, -1.5194628238677979f, -0.5850174427032471f, -0.08793152123689651f, -0.6866653561592102f, 0.4101375937461853f, -1.3473118543624878f, 1.4177610874176025f, -0.467404842376709f, -0.9928666353225708f, -1.9059150218963623f, -2.0202691555023193f, 0.32121846079826355f, -0.5283262729644775f, 0.6973733305931091f, 0.9967362284660339f, -1.3085604906082153f, 0.32580918073654175f, 1.8071491718292236f, -0.27431318163871765f, -0.1326882690191269f, 0.32671934366226196f, -0.8231000304222107f, 0.6738032102584839f, 0.551419734954834f, 1.1634254455566406f, -0.9665390253067017f, -0.6739510893821716f, 0.2892301678657532f, -0.46433013677597046f, 0.8342204093933105f, 1.0237910747528076f, 0.659813642501831f, -1.458429217338562f, -0.7326241135597229f, 1.2922906875610352f, 1.6295740604400635f, -0.8344282507896423f, -0.539470374584198f, 1.4665507078170776f, 2.654942035675049f, 1.609459400177002f, 0.367727130651474f, -0.9192121624946594f, 0.35196706652641296f, -0.15669254958629608f, -1.098692774772644f, -1.2132517099380493f, -0.07032525539398193f, 1.8989076614379883f, 0.4093967378139496f, -1.4458993673324585f, -1.5456795692443848f, -0.4921732246875763f, 0.09215065091848373f, -1.1077474355697632f, 1.0582451820373535f, -0.29416555166244507f, -0.5785751342773438f, 1.212047815322876f, 2.0987792015075684f, -1.5964131355285645f, 0.5674765110015869f, 0.9744538068771362f, 0.27448296546936035f, -0.04056643322110176f, -0.5840954780578613f, -0.31466683745384216f, -0.16959930956363678f, 0.658042311668396f, 0.8539144992828369f, -0.7443782091140747f, -0.1468161642551422f, 1.0819309949874878f, 1.2373405694961548f, -1.3463493585586548f, -0.5285679697990417f, -1.5322294235229492f, 0.8283826112747192f, 0.45322731137275696f, -0.3450125455856323f, -0.2933650016784668f, 0.10613418370485306f, 0.11260014772415161f, 0.02781728096306324f, 0.5361921191215515f, -1.0331605672836304f, 1.0188977718353271f, -0.8913255929946899f, 0.8866236805915833f, 0.8713390231132507f, -0.4831593930721283f, -1.7715193033218384f, 0.8562206029891968f, -3.832531690597534f, -0.9927570819854736f, -0.11961783468723297f, 1.397810459136963f, -0.3067088723182678f, -1.4679176807403564f, 0.0910743772983551f, 1.2350000143051147f, 0.5837976336479187f, -1.8317182064056396f, 0.1937909871339798f, -0.8611426949501038f, -0.522757887840271f, 1.7186238765716553f, 0.0968942642211914f, 0.4322175085544586f, 0.11584769189357758f, -0.2557562291622162f, 1.1861584186553955f, -0.9166569709777832f, 0.8642276525497437f, -1.716816782951355f, 0.6718529462814331f, -0.21214154362678528f, 1.0166347026824951f, 0.2806217670440674f, 1.9681813716888428f, -0.27264782786369324f, 1.2400078773498535f, -0.5684311389923096f, 0.36735019087791443f, 0.24113479256629944f, 1.9915446043014526f, -0.9585699439048767f, -1.2203160524368286f, -0.7793720364570618f, -1.0658658742904663f, -0.5502451658248901f, 0.9851812124252319f, 1.387345790863037f, -0.794191300868988f, 0.6909231543540955f, -0.7403607368469238f, 0.9804211258888245f, -0.905766487121582f, -2.9151322841644287f, 0.47318676114082336f, -0.05469474941492081f, 1.0238094329833984f, -0.5702749490737915f, 0.2909986078739166f, -0.41753849387168884f, -1.0129939317703247f, 0.535078763961792f, -0.7609738707542419f, -0.8248465657234192f, 0.36742645502090454f, 0.7183824777603149f, -0.8665743470191956f, -0.17533333599567413f, 0.768028736114502f, 1.0834310054779053f, 0.5876799821853638f, 0.04771281033754349f, 2.1327197551727295f, 0.19281719624996185f, -0.07964225858449936f, 1.1059718132019043f, -0.9939178228378296f, -1.3392163515090942f, -1.2726119756698608f, 0.3544987440109253f, 0.42930465936660767f, -0.5719326138496399f, -0.30639445781707764f, -0.12276917695999146f, 1.62221360206604f, 0.07731132954359055f, 0.183979332447052f, -0.8637551665306091f, 0.23347528278827667f, 0.3847457766532898f, 1.3200364112854004f, 0.2528532147407532f, 2.9082677364349365f, 1.2357676029205322f, -0.6267421841621399f, -0.28073710203170776f, 2.0288591384887695f, -0.46663370728492737f, -0.5359372496604919f, 0.7414069175720215f, 0.08158037811517715f, 1.2316521406173706f, -0.8425887823104858f, -1.141897201538086f, 1.2004878520965576f, -0.8595814108848572f, -1.5196866989135742f, -0.10754015296697617f, 1.4483332633972168f, -2.0370566844940186f, 1.3712586164474487f, -0.243259459733963f, -0.7683921456336975f, 0.1017654687166214f, -0.35750812292099f, 0.35395190119743347f, 0.20281589031219482f, 2.9194436073303223f, -1.6971954107284546f, -0.8292406797409058f, -1.2668757438659668f, 1.2814747095108032f, -0.29335519671440125f, 0.7705280780792236f, -0.5614521503448486f, 1.4171453714370728f, 0.23598231375217438f, -0.29366371035575867f, -1.2262547016143799f, 0.38033702969551086f, -1.2398927211761475f, -0.5345798134803772f, 0.33024173974990845f, 1.9377914667129517f, 1.0871516466140747f, 1.3149042129516602f, -0.9444983005523682f, 0.1276102066040039f, 2.106604814529419f, -1.1304327249526978f, -0.9464563131332397f, 0.1686755269765854f, -0.446566104888916f, -0.6286743879318237f, -0.11334973573684692f, -1.962886095046997f, -1.523536205291748f, 0.645367443561554f, -0.9533721804618835f, -0.5536230206489563f, 0.6801379919052124f, 1.2470786571502686f, -0.1793764978647232f, 0.6101059317588806f, -0.10875027626752853f, 0.996456503868103f, -0.7419731020927429f, -1.1421011686325073f, -1.1778020858764648f, -0.04226749762892723f, 0.30849045515060425f, -0.7805798053741455f, -0.8158796429634094f, -2.9890658855438232f, 1.5734566450119019f, -1.3029974699020386f, 1.1616994142532349f, -0.25818052887916565f, 0.31204938888549805f, -1.2942535877227783f, 0.608341634273529f, -1.1809622049331665f, 0.15562251210212708f, 0.6069639921188354f, 1.4125137329101562f, -0.27004849910736084f, -0.4410642385482788f, -0.1343953162431717f, 1.1372771263122559f, 0.18370741605758667f, 1.8697383403778076f, 1.0696011781692505f, -0.17657233774662018f, -1.340453863143921f, 0.717379093170166f, 0.6038413643836975f, 0.7894347310066223f, 0.9626035690307617f, -0.25843867659568787f, -0.8331825137138367f, -0.02316661737859249f, 1.4387904405593872f, -0.77969890832901f, -0.3609391152858734f, 0.21925963461399078f, -0.4726978540420532f, -1.0638948678970337f, -0.4561823606491089f, 0.5214431285858154f, -0.5561836957931519f, -0.2302778661251068f, 1.117246150970459f, 0.7824702858924866f, -1.1049346923828125f, -0.7501540780067444f, -0.47921040654182434f, -0.011093219742178917f, 0.03258634731173515f, -0.756567120552063f, 0.104627326130867f, 0.4133651852607727f, -0.958824634552002f, -0.2272576242685318f, 1.3634916543960571f, -2.3571479320526123f, 1.173104166984558f, 0.7147674560546875f, 0.4163925051689148f, -0.06465863436460495f, -0.7909473180770874f, 1.2354754209518433f, 0.2998451292514801f, -0.9952824711799622f, -0.5479734539985657f, 1.514818549156189f, -0.325872004032135f, 0.0061524491757154465f, -1.5123530626296997f, 0.25006136298179626f, -2.4219810962677f, -0.8351162672042847f, 0.6859942674636841f, 1.443666696548462f, 0.6747081875801086f, -0.0002302070497535169f, -0.21609316766262054f, 1.0141247510910034f, 0.721377432346344f, -0.29935362935066223f, -0.49368005990982056f, -1.5796416997909546f, 0.3189202845096588f, -0.5672794580459595f, 1.5067633390426636f, 1.161847710609436f, -0.08308011293411255f, -2.262038469314575f, 0.8626111745834351f, 0.196146160364151f, 0.2249874323606491f, 0.6830949187278748f, 0.0019485306693241f, 1.01323664188385f, -0.6184468865394592f, 0.7670295834541321f, 2.8818862438201904f, 0.6386357545852661f, -0.10977188497781754f, -0.17038141191005707f, -0.938051164150238f, -0.11422315984964371f, 0.1034913882613182f, -1.2966164350509644f, -0.41023650765419006f, -0.8332573175430298f, -0.9556332230567932f, 0.1063748151063919f, -0.20979070663452148f, 0.6346396803855896f, 0.545662522315979f, -0.937210738658905f, -1.1672022342681885f, -0.4361383616924286f, 0.0964970588684082f, -0.3027929663658142f, -0.638506293296814f, -0.05629456788301468f, 0.11661393195390701f, 0.3269132375717163f, -0.7588294148445129f, -0.952683687210083f, -0.7779799103736877f, 0.3503226041793823f, 1.2010222673416138f, 0.8768793344497681f, -0.7691330313682556f, -2.330476999282837f, 1.9413354396820068f, -1.0022987127304077f, -1.4932167530059814f, -1.2868070602416992f, 0.21946865320205688f, 0.8497498631477356f, -0.02522038109600544f, 0.6064606308937073f, 1.6981608867645264f, -0.36366915702819824f, 0.6935068964958191f, 0.12109924107789993f, 0.675596296787262f, 0.8142848610877991f, 0.0793362557888031f, -0.7808834314346313f, -1.163609266281128f, 0.3780289888381958f, 0.5237964987754822f, -1.3662828207015991f, -0.439242959022522f, -0.8599120378494263f, 0.3788713812828064f, -0.46068817377090454f, 0.7436892986297607f, -1.2479252815246582f, -0.4358613193035126f, 0.2984321117401123f, -1.8900177478790283f, 0.19737061858177185f, -0.7584667801856995f, 0.8250064849853516f, -1.5966553688049316f, 0.04276036471128464f, 0.7530690431594849f, -0.04625223949551582f, -0.1391070932149887f, -0.6105663776397705f, 0.24455690383911133f, -0.8800807595252991f, -0.03276701644062996f, -0.09261447191238403f, -0.9824548959732056f, 1.3447729349136353f, 0.19431965053081512f, -0.6367602348327637f, 0.941990315914154f, 1.2206652164459229f, 0.08320564776659012f, 0.5858650803565979f, -0.2219216674566269f, -0.6149584650993347f, -0.018692156299948692f, 1.0629346370697021f, -0.8915789723396301f, -1.4766337871551514f, 1.0600273609161377f, 0.6533640027046204f, -0.9062879085540771f, 1.4614392518997192f, 1.6020095348358154f, -0.8768263459205627f, 0.43399056792259216f, -0.06297583878040314f, -0.9222075343132019f, -1.3687634468078613f, 0.5819101333618164f, 1.2366944551467896f, 1.6560721397399902f, 1.2221823930740356f, 1.4343817234039307f, 0.982934296131134f, 0.003909424878656864f, 0.16369947791099548f, 1.25956130027771f, 1.0566132068634033f, -1.0371782779693604f, -1.5509816408157349f, 1.1234275102615356f, 1.1463638544082642f, -0.3711417317390442f, 1.1168509721755981f, -1.9369032382965088f, -0.8143561482429504f, 1.0661178827285767f, 0.771891176700592f, -1.9291707277297974f, 0.034095875918865204f, -0.6951659917831421f, 1.3456140756607056f, 0.34626761078834534f, 0.8155362606048584f, -0.7486876249313354f, -0.7881835699081421f, 0.5057575702667236f, 1.2215378284454346f, -0.9714295268058777f, 0.27692535519599915f, -1.5334482192993164f, 1.146209955215454f, -0.18189188838005066f, -1.2797164916992188f, -0.5713837146759033f, 1.1689343452453613f, -1.8829604387283325f, -0.369537353515625f, -2.0042972564697266f, -0.8240620493888855f, -0.3844030499458313f, 0.5684375166893005f, -1.137097716331482f, -0.31372663378715515f, 0.15282166004180908f, 1.0580254793167114f, -0.17965702712535858f, -1.178733229637146f, -1.2379671335220337f, -1.5433486700057983f, -0.6671661734580994f, 0.9025653600692749f, 0.304572194814682f, -0.47141557931900024f, 0.005512263625860214f, 0.8932762742042542f, 0.14350247383117676f, 0.7622402906417847f, -0.758182168006897f, -0.7234253287315369f, 0.7250988483428955f, -0.05147926136851311f, -0.6577823162078857f, -0.036672789603471756f, 0.514220118522644f, -0.6020243763923645f, -0.09203547239303589f, -1.7167327404022217f, -0.7002271413803101f, 0.4779217541217804f, 0.46033239364624023f, -0.386881560087204f, -0.0812682956457138f, 0.5568539500236511f, 0.04228341206908226f, -0.3627345860004425f, -1.3895257711410522f, 0.17128385603427887f, -1.2317392826080322f, 0.6718097925186157f, -0.15104348957538605f, 0.3213997781276703f, 0.9548746347427368f, 0.04617787152528763f, 1.7811017036437988f, 0.8509474396705627f, 1.179113507270813f, -0.35717684030532837f, 1.1261653900146484f, 1.2983627319335938f, -0.6900910139083862f, 0.4424906373023987f, 1.1874427795410156f, -0.21165163815021515f, -1.2463988065719604f, 0.9242297410964966f, 0.13773603737354279f, -0.060615699738264084f, 0.18482893705368042f, 0.09389957040548325f, -0.2936820983886719f, 0.36884430050849915f, -1.0105520486831665f, -1.1568186283111572f, 0.04061844199895859f, 1.7561308145523071f, -0.5621538162231445f, 0.19418247044086456f, 0.027558235451579094f, -0.1787608563899994f, -1.4996418952941895f, 0.26971569657325745f, 1.2250828742980957f, -1.170371651649475f, -0.11677189916372299f, -1.35635244846344f, 0.5242968797683716f, 1.0622302293777466f, -0.620258629322052f, -1.7659958600997925f, -0.06464217603206635f, 0.21129246056079865f, 0.825295627117157f, -0.4999869465827942f, 2.0010547637939453f, -0.8498311638832092f, -0.5403968691825867f, 0.628537118434906f, -0.7642602920532227f, -2.773613929748535f, -0.22911065816879272f, -1.0603324174880981f, 1.0186315774917603f, 1.763922929763794f, -1.4548780918121338f, -2.5379507541656494f, 3.445625066757202f, 1.4860185384750366f, 2.268347978591919f, -0.5500510334968567f, -0.12460697442293167f, 1.166976809501648f, 1.2409895658493042f, -0.7066188454627991f, 0.9183834791183472f, -0.02982438914477825f, -0.3448553681373596f, 1.0654057264328003f, -0.15129554271697998f, -0.048379626125097275f, -0.5897568464279175f, 0.09694309532642365f, -1.1129159927368164f, 0.5585124492645264f, -1.7732727527618408f, 0.023851748555898666f, -1.156516194343567f, -0.34847185015678406f, -1.6212294101715088f, -0.8558416366577148f, 0.40576910972595215f, -0.91661536693573f, -1.0765177011489868f, 0.5447612404823303f, 1.1705763339996338f, 0.9127081632614136f, -0.6960206627845764f, -0.7912065982818604f, -0.4341987073421478f, 0.3491472005844116f, -0.9907275438308716f, -1.5203826427459717f, 0.9009959697723389f, 0.07418181002140045f, -0.9035959839820862f, 0.9040606021881104f, 0.25104597210884094f, 0.4702746272087097f, -0.5477657318115234f, 1.522364616394043f, 1.6411151885986328f, -1.3906173706054688f, 0.5155009627342224f, 0.37119555473327637f, -0.015211731195449829f, 0.8483667373657227f, 0.634861409664154f, 0.5293999314308167f, -0.18842656910419464f, 1.757521390914917f, 1.3967915773391724f, -0.5659265518188477f, 0.1281542032957077f, 1.8777929544448853f, -0.3289257884025574f, 0.02351505309343338f, 0.428303986787796f, -0.7096118927001953f, 0.723761260509491f, 1.5117958784103394f, 0.5427215099334717f, 2.0354838371276855f, 1.0238486528396606f, -0.39082857966423035f, -1.2726235389709473f, -0.6254484057426453f, 0.9178156852722168f, 0.3840969204902649f, -0.19802889227867126f, 0.5223742127418518f, 0.05841278284788132f, 0.28558799624443054f, -1.0703928470611572f, 0.9307976365089417f, -0.18846826255321503f, -0.8914596438407898f, 0.12508316338062286f, 0.3772087097167969f, 0.9574744701385498f, -1.9291036128997803f, 0.5764585137367249f, 0.20935998857021332f, 0.5120382905006409f, -0.7815778851509094f, -1.478035569190979f, 0.051756951957941055f, 2.1100034713745117f, 0.020343216136097908f, -0.6159464716911316f, -0.4243218004703522f, -0.5821011662483215f, 0.5247467160224915f, -0.8761705756187439f, 0.48434850573539734f, 0.019129564985632896f, 0.03497670218348503f, -0.35594069957733154f, 1.505028486251831f, -0.8405776023864746f, -0.15094131231307983f, -0.5910253524780273f, 0.4260155260562897f, 0.7726138234138489f, 0.9289091229438782f, -1.0194525718688965f, -0.4821736812591553f, -0.4351550042629242f, 0.3517809212207794f, 1.2222594022750854f, -0.030398759990930557f, -1.0700304508209229f, -0.4833625555038452f, -0.5188538432121277f, -1.1346086263656616f, -1.0445209741592407f, 0.4199317395687103f, 0.8357481956481934f, 0.9475129246711731f, -0.34265488386154175f, -0.9238923788070679f, 0.3848164975643158f, 0.4773769974708557f, -1.2878144979476929f, 0.2491273283958435f, 0.32681646943092346f, 1.0925219058990479f, -0.7518913149833679f, -1.2260730266571045f, -0.3927396237850189f, -0.7111839056015015f, 0.5547724366188049f, 1.4059346914291382f, -0.22924941778182983f, 0.4364280700683594f, 1.2370060682296753f, 1.1473842859268188f, 0.29434895515441895f, -0.1789979487657547f, 1.9290111064910889f, 1.1900452375411987f, -0.10487240552902222f, -0.5064011812210083f, 2.4400694370269775f, -0.33963847160339355f, 1.7511601448059082f, 1.3401062488555908f, -0.9492565989494324f, -0.8270853161811829f, -1.2403943538665771f, -0.2053070068359375f, -0.2497664988040924f, -0.7054344415664673f, -1.7715047597885132f, 0.4937354028224945f, -0.17678959667682648f, 1.410936951637268f, 2.476804733276367f, -0.8441710472106934f, -1.912861704826355f, 0.007326157763600349f, 0.19589348137378693f, 1.5108956098556519f, 1.0457415580749512f, 1.560778021812439f, 0.24919238686561584f, -1.4739270210266113f, 0.22397132217884064f, 1.1544770002365112f, 0.4349344074726105f, 0.5612083077430725f, 0.6398148536682129f, 0.7980097532272339f, 0.5769069790840149f, -0.22143854200839996f, 0.31076598167419434f, -0.052180398255586624f, 1.7757917642593384f, -1.198473572731018f, -1.1297553777694702f, -1.0952261686325073f, -1.1415528059005737f, 0.5066304802894592f, -0.6875526905059814f, -1.7462857961654663f, 0.06312361359596252f, -1.7986558675765991f, 0.6912683248519897f, 0.49413225054740906f, 0.14730526506900787f, 2.274590015411377f, -1.4762635231018066f, 1.167490005493164f, -0.18522614240646362f, 0.37665531039237976f, -2.800556182861328f, 1.1855055093765259f, 0.7821402549743652f, 0.2779245674610138f, -0.489844411611557f, 0.7103408575057983f, -0.6894504427909851f, -0.6078459620475769f, 0.5198960900306702f, -0.18495284020900726f, -1.3136197328567505f, -0.783242404460907f, -0.47829392552375793f, 1.730208158493042f, 1.0595130920410156f, -0.251127690076828f, 1.2796905040740967f, 1.1880545616149902f, 0.010817415080964565f, 1.351500153541565f, 0.399687796831131f, -0.10175959765911102f, -0.2829807698726654f, -0.5876336097717285f, -0.7546091079711914f, -1.0380902290344238f, -0.6061143279075623f, -0.26778313517570496f, -0.11727923899888992f, 2.056168794631958f, 1.2978315353393555f, -0.527733564376831f, 0.2199985533952713f, 2.055619716644287f, 0.3816293179988861f, -0.8949641585350037f, 0.6981958150863647f, -0.9663392901420593f, 0.6600029468536377f, -0.3451770842075348f, -0.033411234617233276f, -0.10130416601896286f, 0.6449755430221558f, 1.5684525966644287f, -0.630508303642273f, -0.23472319543361664f, 0.3285580277442932f, -0.47698524594306946f, 0.3249095380306244f, 0.6602651476860046f, -1.652677059173584f, -1.596110463142395f, 0.05211031809449196f, -1.4231536388397217f, -0.16193436086177826f, -0.7287773489952087f, -0.8276495337486267f, -0.9271824359893799f, -0.24771355092525482f, -0.283514142036438f, -1.265464186668396f, -1.345624327659607f, 0.14869070053100586f, -0.5625298023223877f, 1.319007396697998f, -1.1078379154205322f, -0.38139888644218445f, 0.673722505569458f, 0.2882064878940582f, -0.7846477627754211f, 0.886084258556366f, -1.4957531690597534f, -0.3524166941642761f, -0.6002296209335327f, -0.5345580577850342f, -1.4335700273513794f, -0.06124546006321907f, -1.0124340057373047f, 0.7134581804275513f, -0.3809734880924225f, 1.6037938594818115f, -0.6744462847709656f, 0.03632983937859535f, 0.2812545895576477f, 2.2968475818634033f, 1.8312735557556152f, 0.0294801015406847f, 0.7718657851219177f, -0.6380594372749329f, -0.05797451362013817f, -1.4051520824432373f, -0.8756160736083984f, -0.9422666430473328f, -0.14667245745658875f, -1.9506268501281738f, -1.632828712463379f, 0.7152358293533325f, -1.8892930746078491f, 2.4672696590423584f, 0.08700139075517654f, 1.649484395980835f, -0.6159215569496155f, -0.5199353098869324f, 0.01673818565905094f, 0.5336771607398987f, 0.2974875271320343f, 1.8911962509155273f, 0.009384111501276493f, 1.4318883419036865f, -0.49658793210983276f, -0.6031076312065125f, -0.32575365900993347f, 1.9036880731582642f, -1.842443585395813f, -0.16547387838363647f, -0.25706610083580017f, 1.3179066181182861f, -0.6072822213172913f, -0.16537998616695404f, 0.6803566813468933f, 0.1680675745010376f, 1.6327744722366333f, 1.3595441579818726f, 0.6304294466972351f, 0.13153165578842163f, 0.36382463574409485f, -0.7942026853561401f, 2.110002040863037f, -1.1197839975357056f, 0.13227719068527222f, -0.3069077730178833f, 2.2180261611938477f, 0.7556464672088623f, -2.05973744392395f, -0.27732527256011963f, -0.12983323633670807f, -1.0805774927139282f, -1.4954025745391846f, -0.8487699627876282f, 0.8632985949516296f, -0.18222470581531525f, 0.3396982252597809f, 0.023615580052137375f, -0.5582384467124939f, -2.211786985397339f, -0.792870283126831f, 1.4188532829284668f, 1.240154504776001f, 1.2471561431884766f, 1.5289140939712524f, -0.24465647339820862f, 0.09731336683034897f, 0.8652670383453369f, -0.29969948530197144f, -1.841177225112915f, 1.1419644355773926f, 0.3660371005535126f, 1.273138165473938f, -0.7165514230728149f, 0.3878994286060333f, -0.5754631757736206f, 1.2297093868255615f, -0.4566229581832886f, -0.6573424935340881f, 0.7881364822387695f, 0.3378683924674988f, -1.988023042678833f, 0.7956935167312622f, 0.9823195338249207f, -0.4144572615623474f, 1.3572540283203125f, 0.3109254240989685f, -0.6753576993942261f, -1.5371414422988892f, 0.9313072562217712f, -2.2768142223358154f, 0.4319401681423187f, 0.07773400843143463f, -1.5976078510284424f, 1.848445177078247f, 1.5493229627609253f, 0.1915358453989029f, -1.266395926475525f, -2.1607372760772705f, 0.7240417003631592f, -0.7832543253898621f, 1.1483440399169922f, -0.10642467439174652f, -0.32092228531837463f, 1.0402002334594727f, 0.11275313794612885f, 0.37306782603263855f, 0.8337973356246948f, 1.8036106824874878f, 0.7735506892204285f, -1.1966158151626587f, -0.6088662147521973f, 0.16352267563343048f, -0.3072216808795929f, -0.5694004893302917f, 0.13301636278629303f, -1.508280873298645f, -0.8582409024238586f, -0.7953895330429077f, -1.2091906070709229f, -0.7641999125480652f, 0.2154434323310852f, 0.2061154842376709f, 0.3153077960014343f, -0.3388381898403168f, -0.6360014081001282f, -0.45390239357948303f, -2.7026309967041016f, 0.6710167527198792f, 0.8398252725601196f, -2.002260208129883f, -0.6277503967285156f, -1.3664311170578003f, -0.47482022643089294f, -1.1213213205337524f, 0.07767737656831741f, -0.44291654229164124f, -1.2841796875f, -1.0159707069396973f, 1.3361929655075073f, -0.46696820855140686f, -0.2509534955024719f, 1.4244478940963745f, -0.6108700037002563f, -0.4096096158027649f, -0.4688802659511566f, -1.2304245233535767f, -0.14591790735721588f, 0.9639950394630432f, -0.9581076502799988f, 0.8579769134521484f, 0.17619068920612335f, -0.009090420790016651f, 2.2291040420532227f, -0.599851667881012f, 0.5864706635475159f, -0.40187644958496094f, 0.7005379796028137f, 2.269195318222046f, 1.1897660493850708f, -0.530225396156311f, 0.22658583521842957f, 0.8770406246185303f, -0.42359691858291626f, 0.04045804589986801f, -0.3348093032836914f, -1.6933249235153198f, 0.578863799571991f, 0.253021240234375f, 0.8249357342720032f, 0.17797666788101196f, 0.21032211184501648f, -1.3109723329544067f, 1.4387873411178589f, 1.3104835748672485f, -0.4802001118659973f, 0.2532861530780792f, 0.34186819195747375f, -2.0921499729156494f, -1.4881683588027954f, 0.24693161249160767f, -1.7462626695632935f, 0.28935855627059937f, 1.5511679649353027f, 0.8895691633224487f, 0.4553866684436798f, -1.143309235572815f, -0.06552600860595703f, 0.03079218976199627f, -1.0684497356414795f, -0.3178943395614624f, 2.2536263465881348f, -0.19900000095367432f, 0.5933657884597778f, 1.5937228202819824f, 0.8582064509391785f, 0.11251302808523178f, 0.2912493050098419f, 0.12979160249233246f, 0.08444352447986603f, 2.015190601348877f, 0.5599141120910645f, 1.3394620418548584f, 0.6110420823097229f, -0.5921895503997803f, -0.16634242236614227f, -0.3773656487464905f, 1.1717573404312134f, 0.6101416945457458f, 1.917343020439148f, 2.563725471496582f, 3.035121202468872f, -0.534939169883728f, -0.4571112394332886f, 0.802577018737793f, -0.28734639286994934f, 0.12288182973861694f, 0.21209175884723663f, -0.996752142906189f, 0.6648997068405151f, -1.1771180629730225f, 0.517611563205719f, 2.260444164276123f, 0.8793008327484131f, -1.4391361474990845f, -0.47865501046180725f, -0.12678508460521698f, -1.1487538814544678f, 0.9171100854873657f, 0.9602430462837219f, 0.20192822813987732f, -0.6621134877204895f, 0.283156156539917f, 1.1861251592636108f, -0.6021904945373535f, -0.2499617040157318f, 1.7409440279006958f, -0.7325262427330017f, -0.3309515118598938f, -0.7796709537506104f, 1.6620672941207886f, -0.0578240342438221f, 0.2314254492521286f, 0.22709836065769196f, 2.065865993499756f, 0.4171837866306305f, 0.3846955895423889f, -0.01396828331053257f, 0.5790372490882874f, 0.2881997525691986f, 0.8778835535049438f, 0.5851041674613953f, -0.29608187079429626f, 0.33585336804389954f, -0.7194162011146545f, -0.07809803634881973f, 0.35557520389556885f, -1.7239394187927246f, 0.7924410104751587f, 0.030582552775740623f, 1.0178154706954956f, 1.6657739877700806f, 0.17066167294979095f, -0.2777872681617737f, -0.19989891350269318f, 1.6998308897018433f, 0.7318398952484131f, 1.2774115800857544f, -0.3473694622516632f, -0.7603669166564941f, 1.2199066877365112f, -0.3723330795764923f, -1.8119957447052002f, -0.9909027814865112f, -0.3076462149620056f, 0.01513738464564085f, -1.9378867149353027f, -1.8149548768997192f, 1.3613444566726685f, -0.7856798768043518f, -2.4690840244293213f, 2.432499408721924f, 0.27381452918052673f, 0.8801257610321045f, -0.49671268463134766f, 0.056602153927087784f, 1.435631513595581f, 0.36377331614494324f, 0.46456536650657654f, 1.9551998376846313f, 0.6760215759277344f, 1.1773208379745483f, -0.5477144718170166f, -0.48883718252182007f, -0.2574329078197479f, 0.46062397956848145f, -1.6101100444793701f, 0.2585121691226959f, 1.7188366651535034f, 0.5200220942497253f, -1.301006555557251f, -1.5038570165634155f, -0.31398269534111023f, 1.2563191652297974f, -0.5480050444602966f, -0.06532658636569977f, 2.6805763244628906f, -0.9649611115455627f, -0.18265697360038757f, 0.3190588653087616f, 0.006673657335340977f, 0.566218376159668f, 1.7436981201171875f, 1.3330131769180298f, 1.3450311422348022f, -1.0255969762802124f, 1.3098556995391846f, -0.4485261142253876f, 0.8979019522666931f, -0.1220834031701088f, -1.0596239566802979f, 0.14630116522312164f, -0.870776891708374f, -0.24674776196479797f, -1.2373387813568115f, 0.8207234740257263f, -0.5435441136360168f, 1.1541569232940674f, -0.38771313428878784f, 0.12649448215961456f, -1.2653104066848755f, 1.7627143859863281f, -0.2666304111480713f, 0.5257315635681152f, 0.6358874440193176f, 0.10120877623558044f, 0.17400647699832916f, 1.1357200145721436f, 0.036105670034885406f, -0.6797627210617065f, -0.16857317090034485f, 0.26855215430259705f, -1.4135205745697021f, -2.3971786499023438f, 0.7723278403282166f, -0.15770527720451355f, 0.45117413997650146f, -2.815762519836426f, 0.19697873294353485f, 0.2619018256664276f, -0.8475749492645264f, 0.47121289372444153f, 0.3821605145931244f, -0.2688539922237396f, 1.0989755392074585f, -1.0098387002944946f, 0.688886284828186f, 0.5349103808403015f, 2.787760019302368f, 0.38038408756256104f, 0.22671592235565186f, 0.5783246755599976f, 0.6597211360931396f, -0.28375279903411865f, -0.6992143392562866f, 0.7167068123817444f, -0.09313177317380905f, 0.6840168237686157f, -0.19578158855438232f, -0.9126741886138916f, -0.2800213098526001f, -0.3882693350315094f, 0.10380307585000992f, -1.9663856029510498f, -1.013744592666626f, -1.0242971181869507f, 0.8466978073120117f, 1.2233283519744873f, -0.6524525284767151f, -0.6867133378982544f, 1.1395643949508667f, -0.696540355682373f, -0.39359593391418457f, 0.5099198222160339f, -0.15132363140583038f, 0.6866339445114136f, 0.5310931205749512f, -1.3795819282531738f, -0.6988955140113831f, 0.3475550413131714f, 1.001903772354126f, 0.7994512319564819f, 0.8503185510635376f, 0.791765034198761f, -0.8879274725914001f, 1.02428138256073f, 0.1911749392747879f, 0.8436497449874878f, -0.02484050951898098f, -0.7801733613014221f, 0.625577986240387f, 1.5644149780273438f, 0.5320438146591187f, 1.0700255632400513f, -0.5013832449913025f, -0.3839770257472992f, 0.8672234416007996f, 0.8700053095817566f, 1.8044092655181885f, 0.5074418187141418f, -1.0803741216659546f, 1.7308460474014282f, -0.009462441317737103f, 1.9249420166015625f, -0.3632628321647644f, 0.6629142165184021f, -0.6219039559364319f, 1.013214111328125f, -1.5853021144866943f, -0.9035424590110779f, -0.1714278757572174f, 0.9996393918991089f, 0.2449631243944168f, 0.07007400691509247f, 1.4646685123443604f, 0.4391697347164154f, 1.424518346786499f, -0.32468941807746887f, 0.35460788011550903f, -0.3405316174030304f, -0.6941221952438354f, 0.6556671857833862f, -1.087321400642395f, -1.1486494541168213f, 2.4220407009124756f, 0.7684311866760254f, -1.5410068035125732f, -0.411981463432312f, 1.3056882619857788f, -0.338508278131485f, 0.2913444936275482f, -0.8357943892478943f, -1.1213253736495972f, 0.3143851161003113f, -0.42382940649986267f, -0.43293496966362f, -0.9816297292709351f, 0.058465536683797836f, -1.3252078294754028f, -0.7915613055229187f, 0.47723037004470825f, 0.43918681144714355f, -0.3302071988582611f, 1.7920981645584106f, -1.0610337257385254f, -0.09547755867242813f, -1.1624996662139893f, -0.6900646686553955f, 0.3918372392654419f, -0.0669134333729744f, 1.0711755752563477f, 1.3083903789520264f, -0.05556102097034454f, 0.7882350087165833f, 0.37722504138946533f, -0.3213580846786499f, 0.5956750512123108f, -0.5032515525817871f, -0.7394251227378845f, -0.7672699093818665f, -0.040719106793403625f, 0.15840405225753784f, -0.4783642590045929f, 0.4646506905555725f, -0.49036842584609985f, -0.6546841263771057f, 2.7124791145324707f, 0.4293099343776703f, -1.0469098091125488f, -1.0857888460159302f, -0.05841507390141487f, 0.5456286668777466f, 0.2792662978172302f, 2.1168277263641357f, -1.5103271007537842f, -1.307977557182312f, -1.9681334495544434f, -0.8398545384407043f, 1.3870203495025635f, 0.9826740026473999f, -0.2890629470348358f, 1.0335131883621216f, -0.19352838397026062f, 0.07122802734375f, -0.16152557730674744f, 1.0519514083862305f, -1.4766151905059814f, -1.2671267986297607f, 0.2393292933702469f, 1.2190254926681519f, 0.17837846279144287f, -1.3159326314926147f, 0.646186888217926f, 1.6116095781326294f, -0.05114508792757988f, 0.41359904408454895f, -0.20463405549526215f, -2.1418421268463135f, 1.7502615451812744f, -1.4018466472625732f, -0.5717736482620239f, -0.38931718468666077f, -0.9860185384750366f, 0.5779818296432495f, 0.4738202393054962f, -0.7853313088417053f, -0.1793491244316101f, -0.1285407692193985f, 0.9474418759346008f, -0.07980115711688995f, -0.6013848185539246f, -0.5239112377166748f, -0.1949448138475418f, 0.7724379897117615f, -0.11171314865350723f, 0.5611289739608765f, -1.5560880899429321f, 1.4754453897476196f, 1.4865750074386597f, -0.02104124240577221f, 0.031055578961968422f, 1.0900859832763672f, -0.5583686828613281f, 1.3067461252212524f, -0.9643430113792419f, 1.6326349973678589f, -1.144444465637207f, 0.13422854244709015f, 1.668276309967041f, -0.6358500719070435f, -0.8220173120498657f, 1.151316523551941f, -0.9632699489593506f, -0.17086558043956757f, 0.14713206887245178f, -0.13801060616970062f, -0.1489359736442566f, -0.06645472347736359f, -0.880412757396698f, 0.1913401335477829f, -0.05544678866863251f, -0.2204407900571823f, -1.1233654022216797f, -1.1245696544647217f, 0.674127995967865f, 0.2520127594470978f, 0.7975156307220459f, 0.698857843875885f, -0.4366423487663269f, -2.0883853435516357f, -1.366032361984253f, 0.09937097132205963f, -0.3652037978172302f, 1.2572640180587769f, -0.8791263699531555f, 1.3767470121383667f, 0.6487414836883545f, -0.22251465916633606f, -0.7945832014083862f, 1.3094075918197632f, 1.1708104610443115f, 0.9832903742790222f, -0.7685240507125854f, -0.5897563099861145f, -0.008548072539269924f, 0.7963455319404602f, -0.6708536148071289f, -0.16284485161304474f, -1.8156386613845825f, 0.1582425981760025f, 1.5490630865097046f, 0.6892044544219971f, -1.5244767665863037f, 0.1314752995967865f, -0.5694751739501953f, -0.07285541296005249f, 0.02161409705877304f, 0.27252840995788574f, 1.5375657081604004f, -0.16460900008678436f, 0.04459971562027931f, 0.4962165355682373f, 0.9521092176437378f, 0.18977606296539307f, 1.1129406690597534f, -1.742975115776062f, 0.015372429974377155f, 0.43921586871147156f, -0.4629494845867157f, 0.683012068271637f, 0.0549420528113842f, 1.1227099895477295f, -0.9156265258789062f, 0.7202309370040894f, -1.7771120071411133f, -0.49309614300727844f, -0.3537367880344391f, 0.6029451489448547f, 1.4749457836151123f, -1.2571945190429688f, 1.1716068983078003f, -1.2938947677612305f, -0.027610722929239273f, -0.04813985526561737f, -1.6245776414871216f, -0.1380276381969452f, -0.7646363377571106f, -0.24534782767295837f, 0.21846671402454376f, 0.8769089579582214f, -1.0646121501922607f, 0.50408536195755f, 0.15752233564853668f, -0.5226418375968933f, -1.4756296873092651f, 0.25705486536026f, -1.7179490327835083f, 1.3074569702148438f, -0.4480331838130951f, 1.1221709251403809f, 1.4023377895355225f, 1.4818451404571533f, -1.4184188842773438f, 0.2722645699977875f, 1.2556324005126953f, -1.6670103073120117f, 1.050782322883606f, 0.13771948218345642f, -1.1567072868347168f, 1.0360978841781616f, -0.8660301566123962f, -1.0625808238983154f, 1.0239890813827515f, 0.7085686922073364f, 1.8764188289642334f, 2.794304609298706f, 0.38867735862731934f, 1.6592570543289185f, -0.8973153829574585f, -0.428587943315506f, -0.8077448010444641f, -2.2226719856262207f, 1.3841030597686768f, 0.27507975697517395f, 1.8155642747879028f, 0.5318379998207092f, 1.278135895729065f, -0.632580041885376f, -1.036582350730896f, 0.294900506734848f, -0.8918796181678772f, -0.08464081585407257f, 0.7254555821418762f, 0.40998774766921997f, -0.0699654296040535f, 0.8239985108375549f, 2.0619850158691406f, -1.0847980976104736f, -1.5027097463607788f, 0.4682971239089966f, -2.0921216011047363f, -0.31479185819625854f, 0.35259687900543213f, -0.6293240189552307f, -1.997800350189209f, -0.6937687397003174f, 0.43931636214256287f, -0.2434539496898651f, -1.389511227607727f, -0.0009331775945611298f, 0.4944293797016144f, -1.3989880084991455f, 0.3048802316188812f, 0.612401008605957f, -1.0865437984466553f, -0.7029951810836792f, -0.651735246181488f, 0.02101830020546913f, -0.07500791549682617f, -1.676796793937683f, 1.5087897777557373f, -0.8013211488723755f, -1.030982494354248f, -1.4118940830230713f, -0.8552696704864502f, -0.1083751767873764f, 2.1871159076690674f, -0.6815629601478577f, -1.4610782861709595f, -0.5785004496574402f, 2.1495704650878906f, -0.179605171084404f, 1.1426492929458618f, -0.054547298699617386f, 0.40587425231933594f, 0.6724022030830383f, 0.19652637839317322f, -0.07756809890270233f, -1.4263975620269775f, -1.0503880977630615f, -1.5335310697555542f, 0.4794066250324249f, 1.1617707014083862f, 0.8538435697555542f, -0.20266015827655792f, -0.725740909576416f, -0.9262224435806274f, 0.8973854184150696f, -0.7538284063339233f, -0.8116031885147095f, 0.5351168513298035f, 1.988852858543396f, 1.0685486793518066f, -0.5014570951461792f, 0.7195857763290405f, 1.110618233680725f, 0.1288852095603943f, -1.2922874689102173f, 0.6346917748451233f, 1.5678290128707886f, 0.03146770969033241f, 1.8645329475402832f, -0.8617974519729614f, 0.051693860441446304f, -1.4663174152374268f, -0.26106584072113037f, -0.06878392398357391f, 0.8156912922859192f, 0.4850883483886719f, -2.2269856929779053f, 0.5959717631340027f, 0.21369199454784393f, 0.2891088128089905f, 0.3474847674369812f, -0.43252453207969666f, -0.5997704267501831f, 0.2910449504852295f, -0.8439115881919861f, -0.013285640627145767f, -0.5315346121788025f, 0.08023621141910553f, -0.6925831437110901f, -0.6154842972755432f, -1.4682725667953491f, 0.005962289869785309f, -0.17264695465564728f, -1.3461240530014038f, 0.5067698955535889f, -0.7377328872680664f, -0.006388451438397169f, -1.411285400390625f, 0.23984402418136597f, 0.4191838204860687f, 1.395574688911438f, 0.976121723651886f, 0.40693262219429016f, -0.6316997408866882f, 1.5522570610046387f, 0.2695809602737427f, 1.6630321741104126f, 1.0006709098815918f, -0.6625555157661438f, -0.05369861051440239f, 0.2828640341758728f, 0.8807763457298279f, -0.3764995038509369f, 1.8677223920822144f, 2.1043002605438232f, -0.8154667019844055f, -0.4016047418117523f, -0.07730520516633987f, 0.4082213044166565f, -0.7410086393356323f, -2.3087081909179688f, -0.031602293252944946f, -1.454459547996521f, 0.7048721313476562f, -0.5495360493659973f, -1.0822854042053223f, -1.1814165115356445f, -0.7662639021873474f, 2.26790452003479f, -0.9821573495864868f, -0.7700114250183105f, -0.6950612664222717f, -0.47602152824401855f, -2.168846368789673f, -0.4960733652114868f, 1.8063979148864746f, -2.195838451385498f, -1.2756946086883545f, -0.23146040737628937f, -0.6978404521942139f, 0.05868193879723549f, 0.3181943893432617f, -0.9001671671867371f, 1.4585676193237305f, 0.9991294741630554f, -0.958099365234375f, -1.5130436420440674f, 0.8987486362457275f, 0.6024074554443359f, 1.213675856590271f, -0.9290968179702759f, 0.9378061294555664f, 0.32025381922721863f, -0.6372569799423218f, 0.5549997091293335f, 0.4728609323501587f, 1.5382329225540161f, -2.230900526046753f, 2.1022186279296875f, 1.7723872661590576f, -0.29818224906921387f, 1.774613618850708f, -0.403571754693985f, -0.763476550579071f, -0.13895398378372192f, -1.8085858821868896f, -0.1992855966091156f, -0.38452786207199097f, 0.772721529006958f, -0.761553168296814f, 0.32450324296951294f, -0.6566532254219055f, 1.0444575548171997f, -1.0310343503952026f, 0.13313262164592743f, 0.32965973019599915f, -0.7564722895622253f, 1.5161906480789185f, -0.442477822303772f, -0.19663105905056f, -0.5199072957038879f, 0.19425715506076813f, 0.468250036239624f, 0.658501386642456f, -0.16666415333747864f, -0.4670298099517822f, 1.4936802387237549f, -0.8678151965141296f, -0.26301294565200806f, 0.20588766038417816f, 0.022425031289458275f, 0.6587340831756592f, -0.20476730167865753f, -1.0465848445892334f, -1.2654074430465698f, -0.3633078634738922f, -0.4297706186771393f, 0.6679531335830688f, 1.0863715410232544f, 0.7616772055625916f, -0.011849306523799896f, -1.2028323411941528f, 0.5853366255760193f, -0.10431576520204544f, 0.2191448211669922f, -0.8329556584358215f, -0.2799490988254547f, 0.6428387761116028f, 1.405016303062439f, 1.3759163618087769f, 1.4840714931488037f, -1.5530608892440796f, -0.9330264925956726f, -1.1588698625564575f, -0.48916497826576233f, -0.6450856328010559f, -0.11284743994474411f, -2.4587764739990234f, 0.7599589824676514f, 0.9755558371543884f, 0.05122251436114311f, 0.960921049118042f, -0.6847841143608093f, 0.40178006887435913f, -0.9434769749641418f, -0.23219886422157288f, -0.2043316662311554f, -0.8605437278747559f, -0.3255579173564911f, -0.7162472605705261f, -0.08609210699796677f, -3.6307897567749023f, -0.6838181614875793f, -0.489888072013855f, -1.0136138200759888f, -0.8829219937324524f, 1.1272135972976685f, -1.2231221199035645f, -0.7950332760810852f, 0.6623095273971558f, 2.028413772583008f, 0.8947616219520569f, -1.1885942220687866f, -0.400406152009964f, 0.9303890466690063f, -1.0271012783050537f, 0.60735023021698f, -2.186370372772217f, -1.2513858079910278f, -1.5937029123306274f, -1.391968846321106f, -0.7062578201293945f, 0.5445007681846619f, 0.0782814472913742f, -1.7333787679672241f, -0.8157885074615479f, 0.3931215703487396f, -0.11953603476285934f, 0.14065788686275482f, 1.6809849739074707f, -0.2840443551540375f, -1.4785282611846924f, 0.227834552526474f, 0.26440107822418213f, -0.0759536400437355f, 0.9481480121612549f, 0.8856539726257324f, -1.2799997329711914f, -0.2185828685760498f, -0.15562321245670319f, 1.1858826875686646f, 1.7509973049163818f, 0.4133126139640808f, 0.7854304909706116f, 0.64668208360672f, -0.11213088780641556f, 0.8463886380195618f, 0.045808374881744385f, -0.6186387538909912f, -0.5598823428153992f, 0.3789231479167938f, -0.4264816343784332f, 0.9161636829376221f, 0.13592901825904846f, 0.4121098220348358f, 0.40913498401641846f, 0.4465799629688263f, -0.07119281589984894f, 0.4638987183570862f, -0.5617773532867432f, -0.23070071637630463f, -0.5451535582542419f, 0.018565375357866287f, -0.10694064944982529f, 1.1308897733688354f, 1.4536970853805542f, 0.6201399564743042f, -1.4827039241790771f, 0.4850791096687317f, -0.28107115626335144f, -1.132474660873413f, 0.4989188015460968f, 1.1301037073135376f, -0.3190559148788452f, -0.02982708252966404f, -0.4788385033607483f, 0.3957928717136383f, 0.5253366231918335f, -1.6755516529083252f, 0.35308536887168884f, -0.12078773975372314f, -0.2962888479232788f, -0.08987566083669662f, -0.45044565200805664f, -1.0356383323669434f, 1.7253460884094238f, -2.3891191482543945f, -1.0028913021087646f, -0.3333589434623718f, 0.9801297783851624f, 1.2715272903442383f, -0.7523331046104431f, -0.13584411144256592f, 0.6945744156837463f, -1.9437434673309326f, 0.3301672339439392f, 1.6046744585037231f, -0.4702155888080597f, 1.209662675857544f, 0.8889638781547546f, 0.1620996594429016f, 0.12699493765830994f, 0.7177993655204773f, 0.06786083430051804f, 0.8935588598251343f, 0.11956664174795151f, 1.0848652124404907f, -0.47686854004859924f, 0.2491820603609085f, 0.730522096157074f, 0.09865526109933853f, -0.5309102535247803f, 0.08605138957500458f, -1.499106526374817f, 0.8767158389091492f, -1.1525547504425049f, -0.34564009308815f, -0.8810292482376099f, -1.5831094980239868f, -0.9634820222854614f, -1.0025994777679443f, 0.07062987983226776f, 0.11008060723543167f, 0.7735913395881653f, -0.0040825954638421535f, 0.8591998815536499f, -0.7495615482330322f, -0.38007304072380066f, -1.8807623386383057f, 0.10670913010835648f, 1.1232846975326538f, -0.15919449925422668f, -0.561027467250824f, 0.30757027864456177f, -1.5486440658569336f, -1.054305911064148f, -0.8691359758377075f, -0.06806328892707825f, -0.4993351697921753f, 0.1407848447561264f, 0.07298417389392853f, 1.068840742111206f, -1.1345824003219604f, -0.2619936764240265f, 1.4751683473587036f, 0.1493469625711441f, -0.7424628138542175f, 0.0515340156853199f, -0.4829641580581665f, -0.12765848636627197f, 0.048511698842048645f, -0.6109879612922668f, 1.034902811050415f, 1.2692794799804688f, 1.4105430841445923f, -0.2778305411338806f, -0.2591007351875305f, 1.035352349281311f, 0.4310223162174225f, -0.5226048827171326f, 1.6362823247909546f, 2.1157736778259277f, 0.22881479561328888f, 0.6379362940788269f, 0.4769587814807892f, 0.036591242998838425f, -0.7712311148643494f, 0.11032737791538239f, 1.141363263130188f, 2.2910208702087402f, 0.9097362756729126f, 0.015577469952404499f, 0.7960927486419678f, -1.0888819694519043f, -0.22314855456352234f, -0.42274436354637146f, -0.9645407795906067f, 0.26433172821998596f, -0.4537980556488037f, 1.0802478790283203f, 0.17047974467277527f, 0.5164321660995483f, 0.6420630812644958f, 0.12355764210224152f, -0.6113547086715698f, -0.07968119531869888f, 0.8360456228256226f, -0.9061261415481567f, 0.5791953802108765f, -0.9833644032478333f, 0.04046724736690521f, 0.0772174745798111f, 1.1407610177993774f, -1.1166253089904785f, 0.797391414642334f, -1.2114757299423218f, 0.845003604888916f, 0.917169451713562f, 0.6150916218757629f, -1.4388847351074219f, 0.5669911503791809f, -2.142704963684082f, -0.45466870069503784f, -0.6032977104187012f, -0.13450507819652557f, 0.18595486879348755f, 0.3553427457809448f, 1.6049801111221313f, 0.41566601395606995f, 1.1401103734970093f, 1.4051463603973389f, 0.6006536483764648f, 1.797692060470581f, -0.5229107737541199f, -0.20334042608737946f, -0.45935776829719543f, -0.15298272669315338f, 1.5941132307052612f, -0.1434730738401413f, 0.052168138325214386f, -1.3556065559387207f, 1.5627477169036865f, 1.9382938146591187f, 1.2685400247573853f, 2.026231527328491f, -1.1499669551849365f, -0.8773688077926636f, 0.07689109444618225f, 1.4310468435287476f, -0.18800176680088043f, -1.6620466709136963f, 0.719353199005127f, -0.291658490896225f, 2.5384137630462646f, 0.10889503359794617f, -0.5992977023124695f, -0.394126832485199f, -1.3852146863937378f, -1.254912257194519f, 0.1908181607723236f, -1.0126259326934814f, 0.722631573677063f, -0.34968942403793335f, -0.8847613334655762f, -0.984367311000824f, -0.9932076334953308f, 0.8806517720222473f, -0.09622633457183838f, -1.206941843032837f, -0.49620160460472107f, 0.6914839744567871f, -3.028942346572876f, 1.3682804107666016f, -0.4541301131248474f, 0.45961233973503113f, 0.36240431666374207f, 0.34856998920440674f, -0.821986973285675f, -2.4602458477020264f, 0.37095823884010315f, 0.0116643775254488f, -0.5293033719062805f, -1.6280213594436646f, -0.6806969046592712f, -0.1684180051088333f, 1.5591261386871338f, 0.43387478590011597f, 0.26209768652915955f, 1.2855982780456543f, 0.6979333758354187f, 1.5001028776168823f, -1.1095906496047974f, 0.581849217414856f, 1.3310784101486206f, 1.2475563287734985f, 1.4747735261917114f, 0.3026885390281677f, -1.3104248046875f, -0.5138301253318787f, 0.7339244484901428f, -1.0213230848312378f, -1.202392816543579f, 0.16470061242580414f, 2.572078227996826f, -1.3335460424423218f, 0.546902596950531f, 0.38991880416870117f, 0.15686734020709991f, -0.39347609877586365f, -0.30205461382865906f, 2.149627923965454f, -1.751050591468811f, -0.7567965984344482f, -1.4173346757888794f, 0.5380285382270813f, 0.0939386859536171f, 0.45474377274513245f, -0.6351379156112671f, 0.9299246072769165f, -0.4753718972206116f, 0.9432480931282043f, 0.5524624586105347f, 0.40374335646629333f, 0.6184999346733093f, 0.4535541534423828f, -1.0948821306228638f, -0.320241779088974f, -0.028895698487758636f, 0.7330871224403381f, -1.6074398756027222f, -1.227219581604004f, 1.0834808349609375f, 0.05628223717212677f, 0.20571336150169373f, 0.43112990260124207f, 0.3616485893726349f, -1.1379897594451904f, 0.2402925342321396f, -0.47906389832496643f, 1.4835309982299805f, -1.1785883903503418f, 0.5978415012359619f, 0.5928131341934204f, -0.6638352870941162f, -1.003272533416748f, -0.1671929657459259f, -1.1786675453186035f, 0.8089823126792908f, 0.1908469796180725f, 1.2084790468215942f, -2.24063777923584f, -0.5793269276618958f, -1.1207023859024048f, -1.1295276880264282f, -0.5763978362083435f, -0.26363199949264526f, 0.789819061756134f, -0.4906677007675171f, 0.371157705783844f, -1.484498143196106f, 0.2109839767217636f, -0.8684839606285095f, -1.171597957611084f, -0.9773237109184265f, -0.002813637489452958f, -1.2975432872772217f, 0.3479395806789398f, -0.7253870964050293f, -0.6621626615524292f, -0.2912011444568634f, -2.3239030838012695f, 0.918861985206604f, 1.1420683860778809f, -0.1961705982685089f, -1.5105466842651367f, 0.0809166356921196f, 1.4640601873397827f, -0.4027494490146637f, 1.6002179384231567f, -0.26083904504776f, -0.6423792839050293f, -1.7842353582382202f, -1.4978502988815308f, 1.380509853363037f, 1.877629041671753f, -1.4456785917282104f, -0.28545263409614563f, 0.15021729469299316f, 0.5569579005241394f, -1.6427204608917236f, -0.6389901041984558f, -0.5466802716255188f, -1.0836117267608643f, -1.5072098970413208f, -0.3856680691242218f, 0.9019067287445068f, -0.2348310351371765f, -1.738688588142395f, 0.28999093174934387f, -0.1533069908618927f, -0.9047854542732239f, 0.3279983699321747f, -0.5037038922309875f, -1.9761329889297485f, 0.12128926813602448f, -0.06794015318155289f, 1.1322118043899536f, -1.0514512062072754f, -0.672996997833252f, 0.549104630947113f, -0.32529518008232117f, 0.3177003860473633f, 0.1834784746170044f, 1.7332375049591064f, -0.2832452356815338f, -1.384834885597229f, -0.010661765933036804f, -0.22356857359409332f, 0.5824847221374512f, -0.6592251062393188f, 0.44558772444725037f, -2.2453958988189697f, 0.42556077241897583f, -0.6481807231903076f, -0.25188785791397095f, -1.27206289768219f, 1.1171956062316895f, 1.5053801536560059f, 0.8270988464355469f, 0.7335371971130371f, -0.7486765384674072f, 0.45417457818984985f, -0.3060213625431061f, 0.6338478326797485f, -2.674952745437622f, -0.6763451099395752f, -0.9770270586013794f, 1.5783367156982422f, -0.2843257188796997f, -1.3570187091827393f, -2.2979846000671387f, -0.8282976150512695f, -1.328344464302063f, -0.0004540873342193663f, 0.6817458271980286f, 0.9938647150993347f, -1.4253593683242798f, 0.3879549503326416f, -2.0311243534088135f, 1.2712687253952026f, 0.185286283493042f, 0.07715956121683121f, -0.584597110748291f, -0.5161778330802917f, 0.3677023947238922f, -1.3314623832702637f, 1.5407487154006958f, -0.3439099192619324f, -1.2492866516113281f, -0.8399935364723206f, 0.40630635619163513f, 0.00801965780556202f, -0.3249308466911316f, -0.6664285659790039f, 0.28344160318374634f, 0.5257611274719238f, -1.3125312328338623f, -0.3971971273422241f, -1.5499167442321777f, -0.19803878664970398f, 1.0783737897872925f, 0.5109858512878418f, 0.45442286133766174f, -0.7324386835098267f, -0.1001700609922409f, -0.9963456988334656f, 1.706158995628357f, -0.28345587849617004f, 0.24911366403102875f, -0.5420844554901123f, 0.38604170083999634f, -0.08859143406152725f, -0.7755544781684875f, -0.3989383280277252f, 0.3021530210971832f, 0.8383589386940002f, 0.36416593194007874f, -1.461991786956787f, 0.15854018926620483f, -0.5639234185218811f, 1.251044750213623f, 1.9695827960968018f, 1.165942907333374f, -1.0710505247116089f, -1.4054443836212158f, 2.9302103519439697f, -0.23575513064861298f, 0.08357129991054535f, -0.09462077915668488f, -1.8668535947799683f, -0.34727850556373596f, -0.35426023602485657f, 1.4054851531982422f, 1.4522364139556885f, -0.08649589866399765f, 0.04004557058215141f, -0.10994749516248703f, -0.6241140961647034f, -0.24008060991764069f, -0.39534568786621094f, 0.007234930992126465f, 0.03980718180537224f, -1.2252846956253052f, -0.5507245063781738f, -1.171595811843872f, 1.0313161611557007f, -0.20263050496578217f, 1.3937491178512573f, 1.0930505990982056f, 1.4651625156402588f, 0.2634923756122589f, -0.4779359698295593f, -1.4054735898971558f, 0.7812330722808838f, 0.22484587132930756f, 0.7241055965423584f, -0.6603827476501465f, -0.6927292346954346f, -0.7928091883659363f, -0.4532349705696106f, 0.5121238231658936f, -0.3005983531475067f, -0.44323164224624634f, -0.3902406692504883f, -1.0369137525558472f, -0.19362880289554596f, 0.49349725246429443f, -0.43515950441360474f, -0.6069482564926147f, -1.473711609840393f, -2.3762154579162598f, 1.134980320930481f, -0.9291712641716003f, -0.62174391746521f, 0.47012677788734436f, 1.0591752529144287f, 0.3450745940208435f, -0.04975222051143646f, 1.2706385850906372f, 0.950104296207428f, 0.9791541695594788f, -0.0728548914194107f, 1.3175126314163208f, 0.5881410837173462f, 0.9367974996566772f, 0.9128003120422363f, 0.4015538990497589f, 0.1372641772031784f, 1.041069507598877f, 0.7456277012825012f, 0.0962943509221077f, -1.7508755922317505f, -1.171151041984558f, 1.6682186126708984f, -0.6773730516433716f, -2.147958517074585f, 0.4557051658630371f, 0.061060722917318344f, 0.4539925456047058f, -1.1904879808425903f, 0.2646864056587219f, -0.8139405250549316f, -2.294616460800171f, -0.28324073553085327f, 1.7303272485733032f, -0.46407395601272583f, 1.7161751985549927f, 0.8312178254127502f, 0.2559703290462494f, 0.7086811661720276f, 0.6478610038757324f, -0.06174112856388092f, 0.2679769694805145f, -0.6163668632507324f, -0.4823808968067169f, 0.3113662600517273f, -1.5075656175613403f, -0.32805025577545166f, 0.9543997049331665f, 0.8449776768684387f, -0.8963518738746643f, 0.354059636592865f, -0.15964199602603912f, -0.20036070048809052f, 0.4515315592288971f, 0.987306535243988f, -0.9755827188491821f, 0.2618931531906128f, -2.5518243312835693f, 1.015307068824768f, 2.9806666374206543f, -0.5756134390830994f, -0.42303135991096497f, -1.6033560037612915f, -0.3883368968963623f, -1.6651710271835327f, -1.7486891746520996f, 0.5733898282051086f, -0.17000924050807953f, -1.2242252826690674f, -0.7774103879928589f, -0.662408173084259f, 1.8391462564468384f, 0.4153856635093689f, 0.07132068276405334f, 0.595319926738739f, -0.8092124462127686f, 0.3273322880268097f, -0.4520135819911957f, 0.15657921135425568f, 2.195984125137329f, 0.9013379216194153f, 0.23789522051811218f, -1.9253568649291992f, -0.8211831450462341f, -0.16658450663089752f, -2.5792624950408936f, 1.1836506128311157f, -0.0034865080378949642f, -0.5498691201210022f, 0.3581060767173767f, 0.7652292251586914f, -0.7733491659164429f, 0.6297365427017212f, -0.6815900802612305f, 1.240025281906128f, 0.8412518501281738f, 0.29200538992881775f, -1.805619716644287f, 1.3460670709609985f, 0.02373058721423149f, -0.6370474696159363f, 1.3327691555023193f, -1.63896906375885f, 0.4103865623474121f, 0.25587576627731323f, 1.880447506904602f, 1.9977002143859863f, -1.038170576095581f, 1.879461407661438f, -0.9597489833831787f, -1.338854193687439f, -1.4890133142471313f, 0.9830695986747742f, -0.41451194882392883f, 1.213753581047058f, 0.5297332406044006f, -0.8235613107681274f, 0.3100223243236542f, 0.7781904339790344f, -0.7196370363235474f, 0.8093208074569702f, -0.7462528347969055f, 0.3914867341518402f, 1.1797000169754028f, -2.474013328552246f, 0.43964120745658875f, -0.10444001853466034f, 0.5850116610527039f, -1.6503483057022095f, -1.7112841606140137f, -0.08656994998455048f, -0.3778044879436493f, -0.3667309880256653f, 0.08071848005056381f, -0.865123450756073f, 0.6451484560966492f, -0.5293470621109009f, -1.4819915294647217f, 1.995280146598816f, -0.6921202540397644f, 1.3029942512512207f, -0.1476615071296692f, 0.15694653987884521f, -0.6405912041664124f, -0.8447388410568237f, 0.3645228147506714f, -0.7561046481132507f, 1.417391061782837f, 0.5760944485664368f, 0.13722558319568634f, -0.41890090703964233f, -1.0182898044586182f, 0.448284387588501f, 1.421317458152771f, -0.6346070170402527f, -1.0847539901733398f, -0.5952850580215454f, -0.36421963572502136f, -1.5131834745407104f, -1.9063634872436523f, -1.7777600288391113f, -0.5803453326225281f, -0.6087163090705872f, 0.4910713732242584f, -0.54014652967453f, 1.2756917476654053f, 0.9132878184318542f, -1.101527452468872f, -0.8013262152671814f, -1.9939244985580444f, 1.0690128803253174f, 0.7788016200065613f, 0.35621726512908936f, -0.47195300459861755f, 0.9912806153297424f, -0.21498170495033264f, 1.6797508001327515f, -0.7104259729385376f, 0.06523545831441879f, 0.9159647822380066f, 2.1436846256256104f, 0.4262869358062744f, 0.2694464921951294f, 0.9167301654815674f, 0.6317405700683594f, 0.07572164386510849f, 1.160980463027954f, -0.6461572051048279f, 0.03240058198571205f, -0.7329845428466797f, 0.5573175549507141f, 0.1672649085521698f, -0.30756810307502747f, -0.27301856875419617f, 0.2073671668767929f, 0.1735081523656845f, -0.8468038439750671f, 0.1224641352891922f, 0.020374706014990807f, 1.1085376739501953f, 1.3970056772232056f, -0.5763441920280457f, -1.7790907621383667f, -1.4155789613723755f, -0.11962825059890747f, 1.434804081916809f, -0.6779595613479614f, 0.08594394475221634f, 0.17818263173103333f, 0.8717845678329468f, -0.4843977093696594f, 0.1896892935037613f, 1.0048247575759888f, 0.5574866533279419f, 0.20942160487174988f, -2.0887160301208496f, -1.1161184310913086f, -0.5786607265472412f, 0.8086466789245605f, 0.8664202690124512f, 1.088623046875f, 0.01803939789533615f, 0.968483567237854f, -0.3800842761993408f, 0.3059806525707245f, 0.21581949293613434f, -0.4931473731994629f, -0.15840917825698853f, 0.09949837625026703f, 0.30463987588882446f, 0.9820282459259033f, 2.1544744968414307f, 0.14304663240909576f, -1.460897445678711f, -0.1060129776597023f, -0.37611088156700134f, -0.7917583584785461f, -0.08468178659677505f, 0.8363486528396606f, -1.3825470209121704f, 0.6469754576683044f, 0.5219618082046509f, -0.6058419346809387f, -1.969196081161499f, -0.9246644377708435f, -0.508442759513855f, -1.044813632965088f, 0.353022038936615f, 0.8511560559272766f, -2.0522587299346924f, -0.02700735814869404f, -0.7895724773406982f, -2.2979352474212646f, 1.6485515832901f, -2.076538562774658f, 0.5667335987091064f, 1.3000932931900024f, 0.34126174449920654f, -0.384304404258728f, -0.2445336878299713f, 0.6715037226676941f, -0.3841925263404846f, -0.8124385476112366f, 0.4127030372619629f, 1.1925286054611206f, -1.131065011024475f, 0.3341250419616699f, -0.5277276635169983f, 0.24021567404270172f, -0.18079474568367004f, 0.9263569712638855f, -2.2063262462615967f, 0.6430982351303101f, 1.010899543762207f, -1.5004045963287354f, -0.6776713728904724f, 0.4190317988395691f, -0.1862286925315857f, 0.49416670203208923f, -0.9246350526809692f, 0.5439687371253967f, 1.2584720849990845f, -0.27466103434562683f, -1.1287552118301392f, 0.3993513286113739f, -1.8577489852905273f, 1.882336139678955f, 0.28583550453186035f, -1.3191593885421753f, -0.04691770300269127f, 1.424126148223877f, 0.47822806239128113f, -0.22128069400787354f, -0.16418692469596863f, 1.2590702772140503f, -1.5019725561141968f, -1.2077754735946655f, -0.5713672637939453f, -0.29712751507759094f, -0.08878864347934723f, -1.2249038219451904f, 0.9701720476150513f, 0.02799459546804428f, 2.49943208694458f, -0.5693896412849426f, 0.2992815375328064f, 0.4264316260814667f, -1.8905819654464722f, 0.5416959524154663f, 0.5037822723388672f, 0.06191536784172058f, 0.7051315307617188f, 1.1914323568344116f, 0.7841975092887878f, -0.5143458247184753f, 0.10756650567054749f, 0.03601418063044548f, -1.586909532546997f, -0.36298084259033203f, 0.05457804724574089f, -1.8995989561080933f, -0.4206298589706421f, 1.725341796875f, 0.2990115284919739f, 0.2708130478858948f, 1.202155351638794f, -1.3849509954452515f, 0.37148424983024597f, 1.2834670543670654f, 0.40953299403190613f, 1.5861903429031372f, 2.2006518840789795f, 1.1788570880889893f, 0.04892049357295036f, 0.45504313707351685f, -1.1838287115097046f, 0.2520817816257477f, -1.0377240180969238f, -1.1767886877059937f, 1.5097410678863525f, 0.6526481509208679f, 0.5476107001304626f, -1.4937485456466675f, -0.714340090751648f, -1.4507311582565308f, 0.3059408664703369f, 1.3935821056365967f, -2.0411956310272217f, -1.7664213180541992f, 1.1327776908874512f, 0.7594925165176392f, 0.8204308748245239f, 0.7248584032058716f, 0.6390634179115295f, -0.5571966767311096f, -0.40140989422798157f, -0.22562873363494873f, -0.5097048878669739f, 0.9061921834945679f, -0.15940190851688385f, -0.4242385923862457f, -0.23000852763652802f, -0.5473349094390869f, 0.3962896168231964f, -1.0022584199905396f, -1.9527696371078491f, -0.9625334739685059f, 0.7991358637809753f, -0.010661646723747253f, 0.023252639919519424f, 0.6258861422538757f, 0.3616181015968323f, -0.2700897455215454f, 1.6419692039489746f, -0.7898518443107605f, -0.32855224609375f, 0.9272356629371643f, 0.23767738044261932f, 2.069274663925171f, -0.6138983368873596f, 1.5534266233444214f, 0.562792956829071f, 0.9539335370063782f, 0.34314191341400146f, 0.32635602355003357f, -1.0357731580734253f, -0.021849583834409714f, 0.4444722533226013f, 1.132546067237854f, -1.3696054220199585f, -0.42242521047592163f, 0.6405972242355347f, 0.026323553174734116f, -1.8021960258483887f, -0.17883431911468506f, 0.6420789957046509f, 0.17955924570560455f, -0.2522568106651306f, -1.4123497009277344f, -0.7108830809593201f, -0.39913156628608704f, 0.6917201280593872f, -1.742424488067627f, 0.37175261974334717f, 1.2827181816101074f, -0.1333233267068863f, 2.0086865425109863f, -0.2341606765985489f, 0.7056452035903931f, 0.3473411798477173f, 0.809699535369873f, -3.0800278186798096f, -1.1150798797607422f, 1.4471967220306396f, 0.812851071357727f, 0.4065439999103546f, -0.545810341835022f, -0.6789452433586121f, -0.7672881484031677f, 0.17289385199546814f, 0.40247759222984314f, 0.6123214364051819f, -0.6715307831764221f, 0.18855510652065277f, 1.0615094900131226f, -0.2978079319000244f, -0.4801141619682312f, -1.3616151809692383f, -0.9356100559234619f, 0.978323757648468f, 1.4345933198928833f, 0.956223726272583f, -0.9492759704589844f, -0.9247362613677979f, 1.4304224252700806f, -1.2141680717468262f, 1.5573612451553345f, -2.772749185562134f, -2.07552170753479f, 0.019730882719159126f, -1.6115124225616455f, 0.2347842901945114f, -0.6375554203987122f, -0.2960043251514435f, -0.41874298453330994f, -1.3194098472595215f, 0.05774686485528946f, 0.30748531222343445f, -0.5826377272605896f, -0.31068262457847595f, 0.39234694838523865f, 1.9441745281219482f, 2.0378758907318115f, 0.4360438287258148f, -0.09107904881238937f, -0.24155908823013306f, 0.45299890637397766f, -0.10979556292295456f, 1.179660439491272f, 1.1283513307571411f, -0.8159404993057251f, -0.07127540558576584f, -0.8951465487480164f, 0.31810882687568665f, 0.6084815859794617f, 0.19665859639644623f, -1.3970006704330444f, 1.1369043588638306f, -0.11328133195638657f, 0.12461294233798981f, -0.5991875529289246f, -1.8511996269226074f, 1.1412599086761475f, -1.323111653327942f, -1.642134666442871f, -0.7886322736740112f, 0.016923261806368828f, 0.26853591203689575f, -0.08590231090784073f, -1.8297702074050903f, 1.8586663007736206f, 0.6306189298629761f, -2.0020906925201416f, -0.9767837524414062f, -1.0065470933914185f, 1.5179888010025024f, -0.25139299035072327f, 0.2684777081012726f, 0.2379687875509262f, -1.7342220544815063f, 0.890319287776947f, -0.038704462349414825f, -1.1842435598373413f, -0.6990516185760498f, -0.6046251654624939f, 1.8508250713348389f, -2.0493991374969482f, -1.7228929996490479f, 0.23755493760108948f, 0.9717959761619568f, -0.3798253536224365f, 0.685245931148529f, -1.1771284341812134f, -0.02216072380542755f, -0.1269814372062683f, -0.26579511165618896f, 0.22640825808048248f, -0.760986864566803f, 0.4831203520298004f, 1.5869791507720947f, -0.6875012516975403f, -1.2886475324630737f, 1.2204482555389404f, -0.5210925936698914f, 2.4901089668273926f, 0.15757296979427338f, -0.6784350275993347f, 1.6848376989364624f, -0.762600839138031f, 0.4223840832710266f, 0.05342012271285057f, -1.407547950744629f, 0.7069153189659119f, 0.5444626212120056f, -0.19080407917499542f, 0.14213266968727112f, 0.20559696853160858f, 1.2673413753509521f, 1.375124216079712f, 0.060717396438121796f, 0.31265851855278015f, -0.1293685883283615f, -1.0525683164596558f, -0.5546149015426636f, 1.0222612619400024f, -0.8595789074897766f, 1.0922929048538208f, -0.7460381984710693f, 0.660965621471405f, 0.454408198595047f, -1.4719254970550537f, 0.376058965921402f, -0.7192203998565674f, -0.9688137173652649f, 0.4969656765460968f, 0.42114102840423584f, -0.3883015811443329f, 0.9459248185157776f, 0.22322431206703186f, -0.3906600773334503f, 0.5856817960739136f, 0.2909909188747406f, -1.3024108409881592f, 1.4306825399398804f, 0.4534458816051483f, 0.9106372594833374f, -0.341061532497406f, -0.7916259765625f, -1.1452873945236206f, -0.4609397351741791f, -0.5050092339515686f, 1.0576668977737427f, 0.8928799033164978f, 0.7486283779144287f, 1.0225255489349365f, 1.0325298309326172f, 0.772061824798584f, 0.6529440879821777f, 1.3521305322647095f, 1.9143685102462769f, -1.3935517072677612f, 0.9990993142127991f, 1.297232985496521f, 2.667762517929077f, 0.888896107673645f, 0.8407386541366577f, 1.3103235960006714f, 1.3254046440124512f, 0.7249560952186584f, 1.3489890098571777f, 0.7757742404937744f, 0.16486017405986786f, -1.9583953619003296f, 2.0979738235473633f, -1.1749324798583984f, -1.5510687828063965f, 0.6880179643630981f, -0.27131080627441406f, -2.00603985786438f, -0.14031292498111725f, 0.24766799807548523f, -0.19394119083881378f, -0.08344011753797531f, 1.1331552267074585f, 0.4335167407989502f, 1.659290075302124f, 0.645967423915863f, -0.5915296077728271f, 1.6242163181304932f, -0.9242795705795288f, 1.5664156675338745f, -1.430296540260315f, 0.24254998564720154f, -0.9536606073379517f, 0.2823708951473236f, 0.9416194558143616f, 0.9761032462120056f, -0.14037710428237915f, 0.21578916907310486f, 0.42492198944091797f, 0.9744104146957397f, -0.6820825934410095f, 0.7403110861778259f, -0.014602077193558216f, -1.9035120010375977f, -0.459785133600235f, -0.22833675146102905f, 0.2052573710680008f, 0.060564182698726654f, -0.6736571192741394f, -0.811122477054596f, -0.011842876672744751f}; - -// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g) -// = gemm_Bpack_dram + (nt*(K/4)+g)*64. -static float gemm_Bpack_dram[64*64] __attribute__((section(".data"))) = -{-0.5196930766105652f, 1.8524175882339478f, 1.8365377187728882f, 2.074131727218628f, -0.7373097538948059f, -0.7686780691146851f, -0.05119974538683891f, 1.5985578298568726f, 0.2122737318277359f, 1.1059595346450806f, 1.311963438987732f, 0.424176424741745f, -0.4922901690006256f, 1.6562608480453491f, 0.4111401438713074f, -0.2428770512342453f, 2.081841230392456f, 1.0677173137664795f, -1.4276772737503052f, -0.33180344104766846f, 1.7054011821746826f, 0.6059234738349915f, 1.1122153997421265f, -0.5635794997215271f, -1.364527702331543f, 0.17917323112487793f, 0.5652397871017456f, 0.3271985352039337f, 0.01357425469905138f, 2.691838502883911f, 1.2729166746139526f, -0.6343013048171997f, -0.10098669677972794f, -0.43878018856048584f, -1.9775099754333496f, -0.03591495007276535f, -0.787344753742218f, -0.11653880029916763f, 1.899356484413147f, 0.45238569378852844f, -0.5850009918212891f, 0.17411360144615173f, -0.2031504362821579f, -1.271364688873291f, 0.8729038834571838f, 0.14136412739753723f, -1.4058030843734741f, 0.4083256125450134f, -1.3799183368682861f, -0.21034009754657745f, 0.8919094204902649f, 0.18319325149059296f, 0.5042280554771423f, -1.007485270500183f, -1.3393090963363647f, -0.4502287805080414f, -0.16011619567871094f, -1.608611822128296f, 0.1806594282388687f, 1.864193320274353f, -1.201043963432312f, -0.07190026342868805f, 0.14386875927448273f, -0.1494988203048706f, 0.6289454698562622f, -0.0685209259390831f, -1.2115036249160767f, -0.5285655856132507f, -1.5568547248840332f, -1.2001843452453613f, 0.2736855447292328f, -0.6129935383796692f, 0.9846591353416443f, 1.6213051080703735f, 0.5066256523132324f, -0.126494362950325f, 0.03450252488255501f, 1.3147404193878174f, -0.6877241134643555f, -0.10271778702735901f, -1.401864767074585f, -0.134507417678833f, -0.5799044966697693f, 1.0302584171295166f, 0.16683164238929749f, 0.13005995750427246f, 0.3381371796131134f, 1.025931715965271f, -1.1549469232559204f, 0.21058911085128784f, 0.0752936601638794f, 0.4396262466907501f, 0.029413584619760513f, 0.18214739859104156f, 0.03471093624830246f, -1.8955014944076538f, 0.9624823927879333f, 0.04533285275101662f, -2.03769850730896f, -1.0922635793685913f, -0.17383840680122375f, -1.065432071685791f, 0.926749587059021f, -0.16234397888183594f, 0.36853712797164917f, -0.2603956162929535f, 0.12695534527301788f, -1.7026076316833496f, 0.6422180533409119f, -0.6823133230209351f, -1.6458345651626587f, 0.15697401762008667f, -1.2265124320983887f, 0.7861263155937195f, -2.095715284347534f, 0.4538240432739258f, -0.4390734136104584f, 0.4836112856864929f, 0.9263717532157898f, 0.06930986046791077f, 1.4818024635314941f, -0.9132123589515686f, -1.606928825378418f, -0.5966755747795105f, 0.7912061810493469f, 1.1138908863067627f, 1.4326399564743042f, 0.34999290108680725f, 0.7664128541946411f, 0.4324735403060913f, -1.018404245376587f, 0.012393372133374214f, 0.6695152521133423f, 1.2033629417419434f, 0.8298293352127075f, -0.9191604256629944f, 0.958231508731842f, 0.2724153697490692f, 0.9915789365768433f, 0.3825709819793701f, -0.5029784440994263f, 1.5228251218795776f, -2.502131938934326f, -0.6252055764198303f, -0.5682758688926697f, -1.4246463775634766f, 0.4983586370944977f, -1.3945032358169556f, -2.804393768310547f, 0.4811320900917053f, 0.3296896517276764f, 1.4936070442199707f, 1.691656231880188f, 0.1595890074968338f, 1.024785041809082f, 0.12853768467903137f, -0.46378275752067566f, 1.3842717409133911f, -0.6843701601028442f, -0.3655944764614105f, -0.34340038895606995f, 0.9877837896347046f, 1.3732529878616333f, -0.32952919602394104f, -0.540728747844696f, 0.5175154209136963f, -1.1346107721328735f, 0.4711856544017792f, 1.920310378074646f, 0.4751400351524353f, 0.5673654675483704f, 0.2462792843580246f, 2.888267755508423f, -0.5556330680847168f, 0.017246929928660393f, -0.3958292305469513f, -0.3625917434692383f, 1.179726004600525f, 0.8268131613731384f, 1.457729697227478f, -1.029435634613037f, -1.2957319021224976f, 0.2842133045196533f, -2.20194673538208f, 0.5491127967834473f, -1.2930694818496704f, 0.4510805904865265f, -0.3953653573989868f, 1.7161552906036377f, 0.15392881631851196f, -1.4651801586151123f, -0.5170696377754211f, -0.789655864238739f, -1.313944935798645f, 1.211775779724121f, 0.9332025647163391f, -0.2605237364768982f, 1.5151708126068115f, -1.521173357963562f, -1.1669979095458984f, -2.0334060192108154f, -0.08260460197925568f, 1.4479997158050537f, 0.1535591334104538f, -0.19014781713485718f, -0.9898110032081604f, 0.3790753483772278f, 1.921068549156189f, -0.13450416922569275f, 0.5259649753570557f, 1.2340812683105469f, 0.19075801968574524f, -0.19310960173606873f, 1.5767059326171875f, 0.11904352903366089f, 0.2899794280529022f, -2.0928122997283936f, -0.0447654202580452f, 0.11111800372600555f, 0.38851943612098694f, 0.5478768348693848f, -1.3495665788650513f, -0.44743525981903076f, -0.7099960446357727f, 0.9817765355110168f, 1.0983757972717285f, -0.25671127438545227f, -2.2621774673461914f, 0.16864269971847534f, -1.6143133640289307f, -0.011093960143625736f, 1.2098065614700317f, -1.4476906061172485f, 1.3612878322601318f, -1.3062708377838135f, 1.6407432556152344f, -0.33931660652160645f, 0.13347899913787842f, -0.9141297936439514f, -0.15479597449302673f, -0.699771523475647f, -1.096972107887268f, -0.55166095495224f, -0.3263216018676758f, 0.6205294132232666f, -1.4098610877990723f, -0.41539478302001953f, 3.068681478500366f, 0.3701395094394684f, -0.7284356355667114f, -0.9289583563804626f, 0.8983376026153564f, -0.48385927081108093f, 0.032833296805620193f, -0.10948953032493591f, 0.46667027473449707f, 0.6613953113555908f, -0.03297487273812294f, -0.13994985818862915f, 1.0257868766784668f, -1.9717178344726562f, 0.5998077988624573f, 1.361150860786438f, -0.173064723610878f, 0.7555463910102844f, -0.14636565744876862f, -0.09429822862148285f, 0.29657915234565735f, 0.11924134939908981f, 0.1167483851313591f, -0.5673035979270935f, -1.7532938718795776f, -1.0694472789764404f, 0.08452916890382767f, 0.33887022733688354f, -1.223060965538025f, 0.19821691513061523f, 0.38515371084213257f, -0.07315804064273834f, 0.8780375719070435f, 0.12963341176509857f, -0.6663824319839478f, 0.4921484887599945f, -0.9834228754043579f, -1.2953417301177979f, -1.5002580881118774f, 1.1558160781860352f, -0.7528656125068665f, -1.160732388496399f, -1.486362099647522f, 0.13496126234531403f, 1.354621410369873f, 1.9504122734069824f, 1.0814120769500732f, 0.7843427658081055f, 1.8615505695343018f, 0.6727956533432007f, -0.4322131276130676f, 1.8005059957504272f, 1.3338027000427246f, -0.3184575140476227f, 1.1182889938354492f, 1.8907326459884644f, 0.19988827407360077f, 0.9396787285804749f, 1.0826958417892456f, -0.6519155502319336f, 0.2863307297229767f, 0.10145679116249084f, 0.09791278839111328f, -0.8082833886146545f, 1.441809892654419f, -0.6460452675819397f, 0.04811352491378784f, 1.382570505142212f, -0.6665758490562439f, 0.7876792550086975f, -0.15957334637641907f, -0.10303135216236115f, 0.27407389879226685f, -0.5971476435661316f, -1.0348221063613892f, 0.42101818323135376f, -0.548947811126709f, -1.2540193796157837f, -1.2696188688278198f, 0.6492337584495544f, 0.2340298593044281f, -0.1553155928850174f, -0.4344737231731415f, -1.0962450504302979f, -1.122981309890747f, -1.865479826927185f, -2.1207547187805176f, -0.6719433665275574f, -0.7559810280799866f, 0.1943359375f, 1.0423648357391357f, -1.1257092952728271f, -0.04083564877510071f, 0.7159355282783508f, 0.3418067395687103f, -0.6778577566146851f, 0.17783714830875397f, -0.16209769248962402f, -0.026765741407871246f, -0.33800795674324036f, 0.6160297393798828f, -1.5755101442337036f, -1.1856271028518677f, 0.3281179666519165f, -0.03522142022848129f, -0.5061670541763306f, 0.8149546980857849f, 0.6836134195327759f, -1.2236212491989136f, 0.9091914892196655f, -0.6015433669090271f, 0.8525826930999756f, -0.6866192817687988f, 1.1305443048477173f, -0.3458130359649658f, -0.673958420753479f, -0.16243989765644073f, 1.8315014839172363f, -0.7265217900276184f, -0.44100794196128845f, -1.1563644409179688f, -1.1872515678405762f, -0.83109050989151f, -1.1144185066223145f, 1.2617039680480957f, 0.6814237833023071f, 2.0168917179107666f, 0.481290727853775f, 0.313506156206131f, -0.5229234099388123f, -0.26092126965522766f, -2.14499831199646f, 1.0247118473052979f, 2.1447153091430664f, -0.34384623169898987f, 0.9131002426147461f, -2.4607415199279785f, 1.2518025636672974f, -0.35988521575927734f, -0.43994462490081787f, 1.9879144430160522f, -0.692777693271637f, 0.4603208601474762f, 0.6009213924407959f, 0.8063201308250427f, 0.979621946811676f, 0.9170622229576111f, -0.11666277050971985f, 0.2106197327375412f, 0.5228466987609863f, -1.7686034440994263f, -0.8117372393608093f, -0.877277135848999f, 0.9701831936836243f, -1.7244060039520264f, 1.61734139919281f, -0.9298601746559143f, 0.41942062973976135f, 2.0680532455444336f, -1.2342560291290283f, 2.678502321243286f, 0.3004297614097595f, -1.303833246231079f, 0.7512189745903015f, 0.3729371726512909f, 0.17139992117881775f, 0.5995815396308899f, -0.19726276397705078f, -0.11891679465770721f, -1.0388710498809814f, -0.1431705802679062f, 1.1298640966415405f, 0.587632417678833f, 0.020525068044662476f, 0.12478796392679214f, 0.2550857663154602f, -0.4017896056175232f, -1.7806396484375f, -0.08095061779022217f, -1.759868860244751f, 0.820906937122345f, -0.15008316934108734f, -1.8686583042144775f, -0.6503500938415527f, 0.6934462189674377f, 1.4084579944610596f, -2.2690858840942383f, 0.39256006479263306f, -0.4266415536403656f, 0.36491602659225464f, 0.807181715965271f, -1.5602378845214844f, -0.055706802755594254f, -0.712527871131897f, 0.2629498243331909f, 1.1325268745422363f, 0.8828094601631165f, 0.9763681888580322f, 0.35255447030067444f, -0.5103265047073364f, 0.048219867050647736f, -0.8320447206497192f, 0.9133552312850952f, 0.35530006885528564f, 1.5981814861297607f, 0.6848441362380981f, -0.238090381026268f, -0.8003812432289124f, 0.22974559664726257f, -0.19079573452472687f, -0.364820271730423f, -0.658343493938446f, -1.2190016508102417f, -1.0552600622177124f, -0.9370644688606262f, 0.07260560989379883f, 0.24500113725662231f, 1.690338134765625f, 1.8711999654769897f, 1.2940467596054077f, 0.49941954016685486f, -0.5331865549087524f, -0.26645612716674805f, 0.3932536244392395f, 0.45422324538230896f, 0.4487703740596771f, -0.43473342061042786f, 1.8583357334136963f, 0.35827821493148804f, 0.1845020353794098f, 1.472187876701355f, 0.9056950807571411f, -2.0051839351654053f, -1.232080101966858f, 1.510047435760498f, -0.4955300986766815f, -1.451595664024353f, 1.1304244995117188f, -0.23605898022651672f, -1.4099090099334717f, 0.033967722207307816f, -0.27385643124580383f, 0.052069131284952164f, 0.6194980144500732f, 1.2993793487548828f, 0.37411054968833923f, 0.29060807824134827f, -0.14929574728012085f, 0.16823826730251312f, 1.7945916652679443f, -0.8427640199661255f, -0.776123583316803f, 0.7065202593803406f, 0.020995743572711945f, -0.7148111462593079f, -0.21817894279956818f, 0.6545652151107788f, -1.3288897275924683f, -0.5552407503128052f, 0.869994044303894f, 0.17624177038669586f, 0.10479936003684998f, 1.1782784461975098f, -0.8539279699325562f, 0.18701297044754028f, -0.3077951669692993f, 1.3164128065109253f, 0.7719241976737976f, 2.0046799182891846f, -0.05154263228178024f, -0.09019286930561066f, 0.9585400819778442f, -1.2275644540786743f, 0.35432279109954834f, -0.2604968845844269f, 0.6541568636894226f, -0.43890517950057983f, -0.05930287018418312f, -1.202606201171875f, -1.2769453525543213f, -0.3968345820903778f, 0.5857738256454468f, -0.3441990613937378f, 0.6814432144165039f, 0.772460401058197f, 0.007647526450455189f, -0.25153985619544983f, 0.84389328956604f, -0.2621386647224426f, -0.42434096336364746f, -0.6080525517463684f, 0.011438353918492794f, 0.0011970907216891646f, 0.672935426235199f, -0.6712173819541931f, -0.2299986481666565f, 0.13960936665534973f, 0.6251678466796875f, -1.8366930484771729f, -0.45854899287223816f, 0.49610286951065063f, -0.8524967432022095f, -0.19241304695606232f, 0.47062209248542786f, -0.7075293660163879f, 0.01930052787065506f, 0.7421151995658875f, -2.1299924850463867f, -1.8519654273986816f, -0.789726972579956f, -1.3833003044128418f, -1.1265941858291626f, 0.4917159378528595f, 0.744454026222229f, -0.761397659778595f, 1.7994297742843628f, 0.6710255146026611f, 1.5392361879348755f, -0.8462389707565308f, -1.374268651008606f, -0.29935309290885925f, -1.3537677526474f, -0.9374998211860657f, 0.966241180896759f, 2.2905983924865723f, 0.4570314586162567f, 1.168905258178711f, -0.22257985174655914f, 1.2293546199798584f, -0.4278397262096405f, -0.527760922908783f, -1.798736572265625f, -0.07677789032459259f, 0.781619668006897f, 0.13910506665706635f, -1.7339553833007812f, 0.7391383647918701f, 1.4115384817123413f, 0.38143935799598694f, -0.051372598856687546f, 0.867921769618988f, -1.1712745428085327f, 1.219971776008606f, -0.9404910206794739f, 0.6608859300613403f, 0.49976596236228943f, 0.9063552021980286f, -0.273310124874115f, -0.8444989919662476f, 0.06740748137235641f, -1.1771124601364136f, -2.319305896759033f, 1.0881538391113281f, 1.7532049417495728f, -1.352665662765503f, -1.717094898223877f, 0.539482057094574f, -0.4647720456123352f, 0.15511354804039001f, -0.933184802532196f, 2.0511553287506104f, 1.7320348024368286f, 0.39241987466812134f, -0.3704788386821747f, 0.8811249136924744f, 0.537391722202301f, -0.48424550890922546f, -1.1632671356201172f, 0.3275741636753082f, -1.6853573322296143f, -1.058225393295288f, 1.4543863534927368f, -0.2975543737411499f, -0.8741039633750916f, -1.0332050323486328f, -0.41522952914237976f, 1.5585501194000244f, 2.5905730724334717f, -0.14419972896575928f, -0.47694188356399536f, -0.2289033830165863f, 0.3704476058483124f, 1.024267315864563f, 0.35092905163764954f, -0.11602573096752167f, -0.5128945112228394f, 0.8613727688789368f, 2.617750883102417f, 0.8522545695304871f, 0.3442921042442322f, -0.21941426396369934f, 1.6952100992202759f, 0.7312605381011963f, -2.2600386142730713f, -1.3617544174194336f, 1.5765711069107056f, 0.1465187966823578f, 1.8045871257781982f, 1.3033205270767212f, 0.18405571579933167f, -1.9564915895462036f, 0.1487964242696762f, -0.9366428256034851f, -0.950509250164032f, -0.2993629574775696f, -0.6517760157585144f, 0.2768969237804413f, -0.006574048660695553f, -1.8791542053222656f, -1.5537861585617065f, -0.9000288248062134f, 1.180068850517273f, -1.0849788188934326f, 0.6662015318870544f, -0.7253139615058899f, 1.3528261184692383f, -0.1364641636610031f, 1.1062331199645996f, -0.2775273025035858f, -0.7301269769668579f, -0.9058868288993835f, 0.3395512104034424f, 0.954081654548645f, -0.042208630591630936f, 0.05911993607878685f, -0.3268890678882599f, 0.8507034778594971f, -0.18483993411064148f, -1.1319291591644287f, -0.03408394008874893f, 1.5095572471618652f, -0.11943084001541138f, 1.4642930030822754f, 0.8833335041999817f, 1.1377960443496704f, 0.7793198823928833f, 0.07684005051851273f, -1.4407286643981934f, 0.5401078462600708f, -0.13687263429164886f, -0.4707719087600708f, -0.23642085492610931f, 2.775019645690918f, -0.048772864043712616f, 0.9577808976173401f, -0.12049184739589691f, -0.008940046653151512f, -1.624023199081421f, -0.5534864068031311f, 0.8154556751251221f, -0.03924920782446861f, 1.350463628768921f, -0.04840381070971489f, -0.7114460468292236f, -0.7299661040306091f, -1.4113084077835083f, 0.34884995222091675f, -0.43344026803970337f, 2.270435094833374f, 2.0011656284332275f, -2.1169581413269043f, 0.3513871729373932f, -0.9149423837661743f, -0.28648310899734497f, 1.87372624874115f, 0.07205427438020706f, 0.0007797196158207953f, -0.14003179967403412f, 0.9105306267738342f, -1.0649497509002686f, 0.45754849910736084f, 0.15436618030071259f, -0.744165301322937f, -0.51121985912323f, 0.23932333290576935f, 0.9544809460639954f, 0.16341786086559296f, -0.30999821424484253f, 0.15972714126110077f, 0.8302736878395081f, -1.421786904335022f, -0.09449474513530731f, 0.07646692544221878f, 0.15851546823978424f, -0.38410186767578125f, -1.4129188060760498f, -1.0182603597640991f, 0.4253365993499756f, 0.98757004737854f, 1.1435766220092773f, 0.042469993233680725f, -0.12866179645061493f, -0.3201303780078888f, 1.3020575046539307f, 0.05820586159825325f, -1.411534309387207f, -0.5787767171859741f, -1.1051533222198486f, 1.3300304412841797f, -0.41862916946411133f, -0.43831324577331543f, 1.0452755689620972f, 0.7565826773643494f, 1.3442822694778442f, -0.005769374314695597f, 1.3893702030181885f, -1.459441065788269f, 0.7733556032180786f, 1.281721591949463f, 1.922197699546814f, 1.5146862268447876f, -0.9400398135185242f, 1.7596426010131836f, 0.4703059196472168f, -1.1514455080032349f, -0.6767635941505432f, 0.9793415665626526f, 0.7851172089576721f, 0.9229853749275208f, -1.3558963537216187f, -0.4045674800872803f, -0.022665904834866524f, 0.04516039043664932f, -0.0492362417280674f, 0.688543975353241f, -0.6404033303260803f, -0.8518562912940979f, 0.5739071369171143f, -1.0256696939468384f, 0.5530444383621216f, 1.2585986852645874f, 0.23174597322940826f, -1.7316545248031616f, -0.300735741853714f, 0.8933703303337097f, -0.2885432839393616f, -0.7255284190177917f, 0.17296810448169708f, -1.597967267036438f, -0.8826010227203369f, -0.7285105586051941f, 1.0816147327423096f, 0.689936101436615f, 0.1465923935174942f, -0.2598642110824585f, 1.2287657260894775f, 0.4021441638469696f, 0.29235512018203735f, -1.4843645095825195f, -1.288714051246643f, 0.8229716420173645f, -1.8816033601760864f, -0.24393519759178162f, 1.1026350259780884f, 0.12240499258041382f, -0.6282588839530945f, -0.6169579029083252f, 0.8369725942611694f, -0.21276021003723145f, -1.037753939628601f, -0.5319015383720398f, -1.4479600191116333f, -0.7402247190475464f, -1.9904698133468628f, -0.8090559244155884f, -2.649178981781006f, 0.6452047228813171f, 1.6631805896759033f, -0.6097846031188965f, -0.5651611685752869f, -0.09456146508455276f, 0.2765144407749176f, 0.2960791289806366f, 1.2084519863128662f, -2.0801830291748047f, 0.8232543468475342f, -0.5484461784362793f, 0.34564146399497986f, 1.220015525817871f, -0.43524736166000366f, -1.7363234758377075f, -1.441352128982544f, 1.4536579847335815f, 1.2042756080627441f, -0.9513401389122009f, 1.7534879446029663f, -1.5643774271011353f, 0.2642883360385895f, 0.5893656611442566f, -0.6958070397377014f, -0.9822604656219482f, 0.9073857069015503f, 0.13120052218437195f, 0.3665435016155243f, 0.8943644165992737f, 0.990159809589386f, 1.0685539245605469f, 0.88179612159729f, 0.11932405829429626f, -1.6899914741516113f, -0.6570671200752258f, 1.6827791929244995f, 0.4652342200279236f, -0.9479421973228455f, -0.527360200881958f, -0.17596317827701569f, 1.684232473373413f, -0.005598508287221193f, -1.2838374376296997f, 0.16975225508213043f, 0.8452434539794922f, 0.28028908371925354f, 0.920665442943573f, 1.9825818538665771f, -1.0555790662765503f, -0.7873882055282593f, -0.31694626808166504f, -0.1027197390794754f, -1.1638461351394653f, -0.17423255741596222f, -0.9778326153755188f, 0.07384445518255234f, 0.2161293476819992f, -2.4143142700195312f, 0.6280608177185059f, 0.604034423828125f, -0.06171112880110741f, -0.27243953943252563f, 2.0528199672698975f, -0.6635909676551819f, 0.4586471915245056f, -1.6872135400772095f, 0.03186811879277229f, 2.094301462173462f, -0.38392743468284607f, 0.5313023328781128f, 2.5650343894958496f, -0.06663789600133896f, -0.08388976007699966f, 0.5069190263748169f, -0.9045271277427673f, -0.13790902495384216f, 1.3361674547195435f, 1.065782904624939f, 0.9382249712944031f, -0.5762946009635925f, 0.26595842838287354f, -0.515076756477356f, 1.8256999254226685f, -0.057207878679037094f, -0.2434438019990921f, 0.9149488806724548f, -2.118633508682251f, -1.2563103437423706f, 1.8411842584609985f, 0.8629192113876343f, -1.979547142982483f, 0.4109382629394531f, 0.054416440427303314f, 0.24019105732440948f, -0.3381706476211548f, -1.1024436950683594f, 0.2338137924671173f, -0.4720325171947479f, -1.3284554481506348f, 0.5184157490730286f, -0.48354241251945496f, 1.3702332973480225f, 0.18823130428791046f, -1.2953920364379883f, -0.13154013454914093f, 0.08959710597991943f, -0.44053706526756287f, 0.20576980710029602f, -1.0061860084533691f, 1.1124646663665771f, -0.7451666593551636f, -0.9709926247596741f, -0.005489187315106392f, 0.8581770658493042f, 0.8032210469245911f, -0.15868736803531647f, 1.6983613967895508f, -0.055956143885850906f, -0.45469725131988525f, 1.2942347526550293f, 0.1953326016664505f, -1.775277853012085f, 1.508985161781311f, -0.47618457674980164f, 1.1132714748382568f, 2.190094470977783f, 0.3530663549900055f, 0.4473947584629059f, 0.7191661596298218f, -1.9299876689910889f, 0.9975112676620483f, 1.1845413446426392f, 1.3882135152816772f, 0.1371254324913025f, -0.84199458360672f, -1.6440988779067993f, -0.06394191086292267f, -0.201401948928833f, 0.2995571792125702f, -1.3531256914138794f, 0.8332006335258484f, -0.1304839849472046f, 0.5836395621299744f, -0.38578328490257263f, -1.6579194068908691f, -0.2272949367761612f, 0.6312239170074463f, -1.346675992012024f, 0.6521760821342468f, -1.3508291244506836f, 2.1090853214263916f, -0.8325138688087463f, -0.9411794543266296f, -0.8664024472236633f, 0.3974232077598572f, 0.33468785881996155f, 0.23106975853443146f, -1.2114311456680298f, -1.2591102123260498f, -0.010216501541435719f, 0.9779312610626221f, 0.0748591348528862f, -1.8819596767425537f, 0.27785420417785645f, 0.734195351600647f, -0.37356483936309814f, -0.395193487405777f, -1.2448596954345703f, -0.4259602427482605f, -0.9261072874069214f, 0.33485373854637146f, 1.0248674154281616f, -0.6913897395133972f, -0.5325083136558533f, -2.0061066150665283f, 0.5374981760978699f, 0.4246957302093506f, -0.39820799231529236f, 0.4305093586444855f, 0.7205038070678711f, -0.2832542359828949f, 0.09373100101947784f, 1.657375454902649f, 1.129982829093933f, 0.6332632303237915f, 0.23538991808891296f, 0.08460769802331924f, 1.6273547410964966f, 1.4943406581878662f, 0.4516901969909668f, 0.681415319442749f, -2.738783597946167f, -0.04374435544013977f, -0.13210587203502655f, -0.20210233330726624f, -1.3241872787475586f, -0.1972821205854416f, 0.29198724031448364f, -1.2132630348205566f, 0.7370628118515015f, 0.9276220798492432f, 0.6291629076004028f, -0.23619088530540466f, -0.02293848805129528f, -2.780918836593628f, -0.8419364094734192f, -0.6721378564834595f, 0.13255943357944489f, -1.203613042831421f, 1.333530068397522f, -2.03551983833313f, 0.8634518384933472f, -1.4491990804672241f, 0.3142701983451843f, -1.005286455154419f, -1.3435431718826294f, 1.2677130699157715f, -1.2937111854553223f, -0.7414584755897522f, -0.3299030065536499f, 0.3301123380661011f, 0.9814369082450867f, -1.4912174940109253f, 0.5385298132896423f, 1.3361884355545044f, -0.5637743473052979f, 0.663472592830658f, 0.5684458613395691f, 0.5110347270965576f, 0.9753285646438599f, 1.9773973226547241f, -1.2740811109542847f, -0.7873809337615967f, 2.1380560398101807f, 0.3426303565502167f, 0.9504527449607849f, -1.5345426797866821f, -0.17094235122203827f, 0.9801839590072632f, -0.05862395092844963f, 2.073120594024658f, -0.3976811170578003f, -0.20158079266548157f, 1.4582887887954712f, -0.5315611362457275f, 0.8970864415168762f, -1.1392240524291992f, -1.309956669807434f, -0.0961947962641716f, -2.4108095169067383f, -0.009612545371055603f, -0.6612817049026489f, 0.8028563857078552f, -0.0704915001988411f, -1.5173195600509644f, 0.20034420490264893f, -0.24687924981117249f, 0.23627454042434692f, 0.7618011236190796f, -0.4506336748600006f, 0.3065869212150574f, 0.9810793399810791f, -0.5398741960525513f, -0.42057791352272034f, 0.4069875180721283f, -0.3191024363040924f, 0.8826714158058167f, 0.6646241545677185f, -2.1909663677215576f, 0.4844568967819214f, 0.6883501410484314f, 1.6444298028945923f, -0.09725860506296158f, -0.972576379776001f, 0.9053502678871155f, 0.3767105042934418f, 0.6639789342880249f, 0.05910481512546539f, 0.6989551782608032f, -0.16035781800746918f, -1.092200756072998f, 0.02478332258760929f, 0.4938628077507019f, -0.5733305811882019f, 0.6898083090782166f, 0.6767069697380066f, 0.5416850447654724f, -0.17832504212856293f, -0.7599223852157593f, 0.4881221354007721f, -1.4561625719070435f, 0.5495442748069763f, 0.1979023516178131f, -0.33105704188346863f, -1.3217003345489502f, -0.24350062012672424f, -0.4956028163433075f, 1.6506794691085815f, 0.7990935444831848f, 0.6958364844322205f, -0.9322998523712158f, 0.18523266911506653f, 0.039212681353092194f, -1.0234346389770508f, 0.31960463523864746f, -0.8433935046195984f, -2.1094107627868652f, -0.36244451999664307f, -0.12838764488697052f, -0.30779823660850525f, -1.713757038116455f, 0.23439815640449524f, -0.40255531668663025f, -1.402301549911499f, -0.8660640120506287f, 0.8337363600730896f, 0.1786288470029831f, 1.002273440361023f, -0.676571249961853f, 0.02751591056585312f, -1.0370279550552368f, 1.4250038862228394f, 0.3661484718322754f, 0.2926573157310486f, 0.9981994032859802f, 0.9345365762710571f, 0.08568228036165237f, -0.5905015468597412f, -0.603374183177948f, 0.6896741390228271f, 0.7225849628448486f, -0.4239010512828827f, 1.3261003494262695f, -1.067254662513733f, 0.6243192553520203f, -2.4506609439849854f, 1.3008989095687866f, -1.3570464849472046f, 0.4011816382408142f, -0.4519657790660858f, -0.5720081329345703f, 1.7311065196990967f, 0.40453165769577026f, 0.5910705924034119f, 0.5206778049468994f, 0.26302585005760193f, 1.462548851966858f, 0.806793212890625f, 2.1060609817504883f, 0.1250244528055191f, -1.0741941928863525f, -0.04460187256336212f, -0.7740861177444458f, 1.4476147890090942f, -1.8094489574432373f, 0.40884074568748474f, -0.13771961629390717f, -0.48222219944000244f, -0.9596375823020935f, 0.88399738073349f, -1.1032230854034424f, 0.4613407552242279f, -0.6266279220581055f, -0.059072766453027725f, 1.432356357574463f, -0.3214779198169708f, -0.32065340876579285f, -1.381635308265686f, -0.36659157276153564f, -1.3767681121826172f, -0.6575090289115906f, -1.0047813653945923f, 0.15763555467128754f, -2.729581594467163f, -0.12834908068180084f, -0.4433963894844055f, -1.7998695373535156f, -0.4873202443122864f, -0.360196590423584f, -0.2588382959365845f, 1.3558588027954102f, -0.6274254322052002f, 0.47283434867858887f, -0.6854686141014099f, 1.4168455600738525f, 0.8578910827636719f, 0.8829494118690491f, -0.8793548941612244f, -0.8100994825363159f, 0.6788665652275085f, 0.33716168999671936f, -0.5290454030036926f, 0.9324967265129089f, 0.6762214303016663f, 0.14378660917282104f, 2.18232798576355f, 0.037647929042577744f, 0.21775013208389282f, 2.395190954208374f, 0.4023849368095398f, 1.6874338388442993f, -2.743908405303955f, -1.9040486812591553f, 0.03124547004699707f, 0.3133164942264557f, -1.2002359628677368f, -0.22739538550376892f, -0.23840101063251495f, -0.682720422744751f, 0.46212872862815857f, -0.9165003299713135f, -1.665152907371521f, 0.7060098052024841f, 0.9033956527709961f, -0.3824882507324219f, 0.1978939324617386f, -0.032255567610263824f, -0.9255485534667969f, 1.1699923276901245f, -1.751257061958313f, 0.9016333222389221f, -1.3840047121047974f, -0.8077333569526672f, 1.2065165042877197f, 0.1894015073776245f, -0.6532776355743408f, -0.2359209805727005f, 0.6057451367378235f, -0.3558703064918518f, 1.4644410610198975f, -0.8236719965934753f, -0.3576895296573639f, -0.9380688667297363f, 1.4365062713623047f, 1.5855119228363037f, 0.39503413438796997f, -0.09881415218114853f, -1.8138774633407593f, -0.6933608055114746f, 1.3770830631256104f, 1.0369852781295776f, 0.522045910358429f, -2.008735179901123f, -0.7871567606925964f, 0.07242458313703537f, -1.9583851099014282f, -0.1682676076889038f, -0.09407275170087814f, -1.5021157264709473f, 0.9083237051963806f, -0.627318799495697f, -1.1001535654067993f, 0.8703535795211792f, 0.9143402576446533f, -0.9333758354187012f, -1.9243427515029907f, 0.5441842079162598f, 1.674156904220581f, -0.1457550972700119f, -0.663499116897583f, 1.2121490240097046f, 0.9221742749214172f, 0.5710628032684326f, -0.7049512267112732f, 2.053586959838867f, 0.9570578932762146f, -1.0222344398498535f, -0.5004768371582031f, -0.8111429214477539f, 2.0269458293914795f, -2.568801164627075f, 1.0234973430633545f, -0.5307855606079102f, 2.509202480316162f, 0.9496951103210449f, 1.4498697519302368f, 1.9099774360656738f, -0.41487377882003784f, -2.154376745223999f, -1.1843920946121216f, -0.13271500170230865f, -0.6129738092422485f, 1.0654757022857666f, 0.589937686920166f, -1.330565333366394f, 1.276455044746399f, -1.1509960889816284f, 1.4542961120605469f, 0.21978320181369781f, -0.26515746116638184f, 0.7749262452125549f, 0.12985540926456451f, 1.0439496040344238f, -1.7788974046707153f, 0.3559875190258026f, -0.05403977632522583f, 0.07127834856510162f, -0.8058504462242126f, -0.35074129700660706f, -0.17826782166957855f, -0.2625838816165924f, 0.7513576745986938f, 0.18278615176677704f, -0.9977543950080872f, 0.45807182788848877f, 0.7924372553825378f, -0.9643475413322449f, 0.46591266989707947f, 1.3539252281188965f, -1.459295630455017f, 0.37469375133514404f, 0.7460726499557495f, 0.26590028405189514f, -1.0854178667068481f, -1.5084643363952637f, 1.1576530933380127f, -0.21968594193458557f, 0.4215473532676697f, -0.32665783166885376f, 2.3412764072418213f, -0.5351880192756653f, 1.0548261404037476f, -0.4031357765197754f, 0.6454427242279053f, -0.6314883828163147f, -1.2809436321258545f, 0.009232764132320881f, 1.2330046892166138f, 0.6238497495651245f, 1.5213422775268555f, -0.4976206421852112f, -1.0449823141098022f, -0.27992430329322815f, 1.2101659774780273f, -0.9885204434394836f, 0.350207656621933f, -0.2644238770008087f, -1.0631657838821411f, -0.7474985122680664f, 0.3266002833843231f, -0.49929699301719666f, -1.4715039730072021f, 0.5023646950721741f, 0.4279913902282715f, 1.0040385723114014f, 1.708235263824463f, -0.2667044997215271f, 0.18906556069850922f, 1.0375696420669556f, 0.7793638706207275f, -0.6498270630836487f, -0.48095330595970154f, 0.13058727979660034f, 0.8354679346084595f, 1.6123450994491577f, -0.4939593970775604f, 0.30456098914146423f, 2.1306238174438477f, 0.026654772460460663f, 0.7856671810150146f, 0.8128212690353394f, 0.04477962106466293f, -1.7439521551132202f, -1.1074808835983276f, -0.6043308973312378f, 0.0144581263884902f, -0.2186170518398285f, -1.0052590370178223f, 1.2696419954299927f, 1.212203025817871f, -0.3618496060371399f, 1.1163915395736694f, 0.4140532910823822f, 0.12140734493732452f, 0.836719274520874f, -0.6638585925102234f, -2.0399112701416016f, -0.6206883192062378f, -0.9584940075874329f, -0.45995032787323f, 0.06401844322681427f, -0.4596543312072754f, -0.5983195900917053f, 0.844002366065979f, -0.6386957168579102f, -1.218558430671692f, -0.1842191219329834f, -1.7404849529266357f, -0.43988513946533203f, 0.978778600692749f, 1.003504991531372f, 1.839215874671936f, -0.7032709121704102f, -2.1381378173828125f, 0.3657694160938263f, -0.01325446367263794f, 1.1153861284255981f, -0.4465695023536682f, 1.315346598625183f, 1.038090705871582f, 2.2033896446228027f, -0.907842755317688f, 2.0937135219573975f, -0.9239965677261353f, -0.553339421749115f, 0.09259668737649918f, 1.0563279390335083f, 0.8357012271881104f, 0.39962950348854065f, -0.5224289298057556f, 1.8076858520507812f, 0.973744809627533f, 1.072774887084961f, -1.0264922380447388f, -1.1302908658981323f, -0.2908953130245209f, -1.6867245435714722f, 1.5600812435150146f, 1.8405765295028687f, -0.7730143070220947f, -0.2413354218006134f, 2.3902435302734375f, -0.5483986139297485f, 0.643738329410553f, 0.7495583891868591f, -1.2053273916244507f, 1.4264872074127197f, -1.1080999374389648f, -0.1472785621881485f, -2.2464029788970947f, 0.40276598930358887f, 0.1035822331905365f, 0.3125080168247223f, 0.19359846413135529f, -0.40304437279701233f, 0.7284680604934692f, 1.4359182119369507f, -1.437017560005188f, 0.5239248275756836f, -0.26981568336486816f, 0.017309220507740974f, 1.2778736352920532f, -0.30529654026031494f, 0.7788206934928894f, -0.08426624536514282f, 0.8460555672645569f, 1.1856203079223633f, -0.4465353190898895f, 0.8045015335083008f, 0.830817461013794f, -0.4116531312465668f, -0.19528378546237946f, 1.0318245887756348f, -0.6419956088066101f, -1.068763256072998f, -0.038084980100393295f, -0.8322976231575012f, 0.8178791999816895f, 0.14809109270572662f, -1.4732903242111206f, -0.10355143249034882f, -0.5564566254615784f, 0.3039886951446533f, 0.9035240411758423f, 0.5980544090270996f, -1.4176050424575806f, -0.0560019314289093f, -1.3687875270843506f, -0.5567623376846313f, -0.36739063262939453f, 1.5230790376663208f, 1.4708421230316162f, -0.4366856515407562f, -1.1750401258468628f, 0.24909113347530365f, 1.8527616262435913f, -1.680544137954712f, 0.659866988658905f, -0.5326191782951355f, 0.5515633225440979f, 1.6462424993515015f, 0.2259528934955597f, 0.27357611060142517f, -0.9556253552436829f, 0.43514177203178406f, -1.1005570888519287f, 0.9130839705467224f, -0.45959728956222534f, 2.1901278495788574f, -0.682377278804779f, -0.22877585887908936f, 0.7370786666870117f, -1.690212607383728f, 0.30514323711395264f, -0.0909213200211525f, -1.1586445569992065f, -0.7481539845466614f, 1.1121810674667358f, -0.4421563148498535f, 1.0719655752182007f, 0.17691805958747864f, -1.2427209615707397f, 0.10128959268331528f, 0.19372011721134186f, 0.24058400094509125f, 0.9057638049125671f, 0.11222076416015625f, -0.059362635016441345f, -2.4078524112701416f, -1.2304182052612305f, -0.026510445401072502f, 0.8861871957778931f, -0.7287083268165588f, 1.380319356918335f, 0.6684188842773438f, -1.7624090909957886f, 0.6292073726654053f, -0.7828742265701294f, -0.08022477477788925f, 0.9383800625801086f, -0.4210285246372223f, -0.49148616194725037f, -0.9306524395942688f, -1.267333745956421f, -0.6902195811271667f, 0.23923389613628387f, 1.4039400815963745f, -2.0282845497131348f, -1.0275602340698242f, 0.9761411547660828f, -0.06743653863668442f, 0.06586836278438568f, -0.7352823615074158f, 0.19450226426124573f, -0.7134578824043274f, 0.16818134486675262f, -1.1888638734817505f, -0.37875133752822876f, -0.321153849363327f, 1.3544011116027832f, -0.423284113407135f, -0.9974878430366516f, -0.9902269244194031f, -0.1406998336315155f, 0.8166614174842834f, 1.0789191722869873f, 1.7252999544143677f, 1.1660224199295044f, 0.5868375301361084f, -0.11877616494894028f, -0.34278497099876404f, -0.08240477740764618f, 0.8542949557304382f, -0.9149259924888611f, 0.5388334393501282f, -1.688096284866333f, -1.05665123462677f, -0.33759868144989014f, -0.2788163125514984f, -0.3783758282661438f, -0.2296314239501953f, -1.1831741333007812f, -1.034787893295288f, -0.8083778023719788f, -0.4777121841907501f, 0.8160690665245056f, 0.8076881766319275f, 0.5698519945144653f, -0.4374706447124481f, -0.7118549942970276f, 2.338996648788452f, -0.0368775799870491f, -0.1774366796016693f, 1.5478016138076782f, 1.0848923921585083f, -0.38529422879219055f, -0.5037281513214111f, 0.9991070628166199f, -0.715931236743927f, 0.28394055366516113f, 0.5939877033233643f, 0.31532949209213257f, 0.27048054337501526f, 0.9632524251937866f, -1.2189302444458008f, -0.06929556280374527f, 0.061022937297821045f, 0.822035551071167f, 0.38992002606391907f, -0.49170032143592834f, -1.689696192741394f, 0.05627693980932236f, -0.2405131459236145f, 0.9160073399543762f, 0.7348390221595764f, -0.6902630925178528f, -0.9186923503875732f, -0.6866548657417297f, -0.10339432954788208f, -0.32694968581199646f, 1.2450132369995117f, 0.6069359183311462f, -0.0937698483467102f, -0.004049783106893301f, 2.6462395191192627f, 1.7450222969055176f, 0.2888743579387665f, 1.1111222505569458f, 1.1171294450759888f, -0.7966887354850769f, 1.0229419469833374f, -1.0742470026016235f, -1.4498995542526245f, -0.5231860280036926f, 0.3438241481781006f, 0.20375612378120422f, -1.4713748693466187f, 0.19777953624725342f, -0.3222813904285431f, 0.5210180282592773f, 1.036444902420044f, 0.2102498859167099f, -1.1502959728240967f, -0.25832024216651917f, 0.2814403772354126f, -0.5159256458282471f, 1.522001028060913f, -0.4228188693523407f, 0.5675636529922485f, 0.47951194643974304f, -0.9628922939300537f, 0.16651970148086548f, 0.17922191321849823f, -0.6149380207061768f, 0.884855329990387f, -0.7095723152160645f, 1.5105708837509155f, 0.9143079519271851f, -1.2950124740600586f, 0.9506115317344666f, -1.3415539264678955f, 0.9584245681762695f, -1.834415316581726f, -0.5945284366607666f, -1.6709949970245361f, -1.3442386388778687f, -0.7921901941299438f, 0.1253465712070465f, -0.9972168207168579f, 0.9316890239715576f, -1.098996639251709f, -0.8827964067459106f, -0.03510960936546326f, 0.33106282353401184f, -1.535096526145935f, 1.5791680812835693f, 1.562730073928833f, 1.0339767932891846f, -0.644834041595459f, -0.7956817150115967f, 0.4747835099697113f, 0.6450149416923523f, -2.4694929122924805f, -1.2703917026519775f, 1.4028544425964355f, 0.39471593499183655f, -0.2914227247238159f, -0.04415971040725708f, 0.6573874354362488f, -1.8288997411727905f, 0.515746533870697f, -2.1813249588012695f, -0.7443293333053589f, 0.4601408839225769f, 1.2225193977355957f, -2.047572374343872f, 1.09447181224823f, 0.42030036449432373f, -0.6366481781005859f, -2.1695642471313477f, 1.1429638862609863f, 0.8955739736557007f, 1.6797635555267334f, -0.6834523677825928f, 0.8514192700386047f, -0.8422428369522095f, -1.0746608972549438f, -0.9010616540908813f, -0.6362974643707275f, -1.087014079093933f, -0.028106562793254852f, -0.1531383991241455f, 1.0988678932189941f, -0.03311381861567497f, -0.892038881778717f, -1.3073756694793701f, 0.14662222564220428f, 0.8670119643211365f, 3.2090775966644287f, -1.3412030935287476f, 0.3424357771873474f, 0.19632267951965332f, -0.1660011112689972f, -0.3333166837692261f, 1.0084987878799438f, 0.9129180312156677f, 0.5135545134544373f, -0.8764104843139648f, -1.4057948589324951f, -0.6906246542930603f, 0.3960992097854614f, -0.2924385964870453f, 0.3751489818096161f, -1.1075292825698853f, 0.37509971857070923f, -1.4790199995040894f, 0.5692160725593567f, -1.146249532699585f, -1.904903769493103f, 0.9906712174415588f, -0.9967307448387146f, -2.3042960166931152f, -1.8701173067092896f, 1.6571006774902344f, -0.5442659258842468f, 0.3686337471008301f, -0.17834435403347015f, -0.7094868421554565f, 0.24787208437919617f, 0.7445037364959717f, 1.5205289125442505f, -1.3338558673858643f, -2.0718472003936768f, -1.8923321962356567f, 2.419602632522583f, -0.24423770606517792f, -0.2663561701774597f, -0.6755290031433105f, -1.124692440032959f, 1.8602604866027832f, 0.6256695985794067f, 0.28849464654922485f, 0.8962987065315247f, 0.8840014338493347f, 0.4107360541820526f, -1.6067783832550049f, 1.4667541980743408f, -1.643362045288086f, -0.7528584599494934f, -0.8185959458351135f, 0.02291426621377468f, -0.7729488015174866f, -0.039141006767749786f, 0.30586403608322144f, -0.2691587805747986f, -0.21813122928142548f, 0.3616786003112793f, -1.017099380493164f, 0.7204318046569824f, -0.10523297637701035f, 1.8706086874008179f, 0.8992230892181396f, -0.23518913984298706f, -0.6936088800430298f, 0.38842713832855225f, -0.9864745140075684f, 0.24743980169296265f, 1.3209456205368042f, 0.5116807222366333f, 0.20717298984527588f, 0.423617959022522f, 0.8553113341331482f, -0.7523386478424072f, -0.7315530180931091f, -0.34988582134246826f, 0.14683414995670319f, 0.337516725063324f, -0.5604003667831421f, -0.3334929347038269f, 0.46565955877304077f, -0.831680178642273f, -0.8464822769165039f, -0.38647857308387756f, -0.1513155996799469f, 0.8186390399932861f, -0.25432348251342773f, -0.47535333037376404f, -0.780207633972168f, 0.778557300567627f, 0.0986286848783493f, -0.754118800163269f, 0.9054679274559021f, -1.1227487325668335f, 0.2190994918346405f, -0.09212681651115417f, 1.8318346738815308f, -0.5303875803947449f, -0.8198762536048889f, -0.8232145309448242f, 0.5506033301353455f, -1.1416276693344116f, 1.8735969066619873f, 1.3694992065429688f, -1.257182240486145f, -1.6220338344573975f, -0.5663302540779114f, 0.6658836007118225f, 0.25084203481674194f, 0.7626033425331116f, -0.9630534052848816f, -0.5952125191688538f, -0.5548334121704102f, 1.2073006629943848f, -0.726753830909729f, -0.25774624943733215f, 0.7335841059684753f, -0.6472665667533875f, 0.4566407799720764f, -0.506052553653717f, -0.07941094785928726f, 0.226389080286026f, 1.392283320426941f, -0.1649594008922577f, -0.8620851635932922f, 1.041149377822876f, -0.18962401151657104f, 2.0505990982055664f, -2.2949159145355225f, -0.8339797258377075f, 2.1567959785461426f, 1.0796829462051392f, -0.815654456615448f, 0.7114473581314087f, -0.24212278425693512f, 0.9910399317741394f, 0.551307737827301f, 0.6295360326766968f, 0.1759207546710968f, 1.6339221000671387f, 1.2599411010742188f, -0.6335592269897461f, -0.8656108379364014f, 0.2853967845439911f, -1.4132448434829712f, 1.7862261533737183f, -0.18690960109233856f, 0.45671215653419495f, 0.5091333985328674f, 0.12447594106197357f, -0.7764351963996887f, -0.9075336456298828f, 1.6591986417770386f, -0.9774589538574219f, -1.437660813331604f, -1.0792182683944702f, -1.0316091775894165f, 0.4912571609020233f, -0.9918735027313232f, 0.37739869952201843f, -0.09308940172195435f, -0.3923634886741638f, -1.9273265600204468f, -0.9188506603240967f, 1.127413034439087f, 1.4187533855438232f, -0.30123433470726013f, -1.6901737451553345f, 0.9281692504882812f, -1.123509168624878f, -1.0524919033050537f, -0.4467827379703522f, 0.523188054561615f, -0.3387533724308014f, -1.7577482461929321f, -0.6524283289909363f, -0.24884779751300812f, -1.2434697151184082f, 1.6212447881698608f, 0.15123429894447327f, 1.4580625295639038f, 0.8593719601631165f, 0.4116728901863098f, -0.3829646706581116f, 0.20710650086402893f, -0.845027506351471f, 0.2616454064846039f, -0.790607750415802f, -0.35345759987831116f, -0.4279586672782898f, 1.68895423412323f, 1.2128044366836548f, 0.9204495549201965f, 0.047616202384233475f, 0.7502005696296692f, -1.6819013357162476f, -0.036870624870061874f, -0.05854296684265137f, 0.6686491370201111f, -0.4365732967853546f, -2.3380324840545654f, 1.8583958148956299f, -1.7244455814361572f, -0.2750453054904938f, 0.4532167315483093f, 1.2953895330429077f, 0.058747299015522f, -0.8122621774673462f, 1.9493451118469238f, 0.7995834350585938f, -0.010960266925394535f, -0.13873475790023804f, -1.9486329555511475f, -0.7052865028381348f, -0.9286441802978516f, 0.5382292866706848f, -1.931458592414856f, 0.17587143182754517f, 1.2172441482543945f, 0.9002020955085754f, 1.1507850885391235f, 0.027491796761751175f, -1.4959322214126587f, -0.18092377483844757f, 0.905379056930542f, 1.720672845840454f, 0.3225809633731842f, 0.5499839782714844f, -0.6421884298324585f, 0.22866190969944f, -0.5370370149612427f, 0.5335260629653931f, -1.0596036911010742f, -2.2213568687438965f, 0.7067712545394897f, -0.21388542652130127f, -0.4895268678665161f, 0.5013054013252258f, -0.9222452044487f, 0.15941572189331055f, -1.377753734588623f, 1.15412437915802f, -1.092998743057251f, -0.8713886141777039f, 0.22347310185432434f, 0.2528358995914459f, 0.21688957512378693f, 0.4699849486351013f, 0.3267545700073242f, -0.09539508819580078f, -0.18269990384578705f, 0.3382812440395355f, -0.6900221109390259f, 0.047023314982652664f, 0.5487117171287537f, -1.9950600862503052f, -0.1581975668668747f, -0.1996922492980957f, 0.46645262837409973f, 0.5603392124176025f, 0.9167394638061523f, -1.983751893043518f, -0.015853818506002426f, 0.033756233751773834f, -1.4661213159561157f, -0.32681238651275635f, -0.5929558873176575f, 0.8652521371841431f, 0.4256573021411896f, -0.2335444986820221f, -0.7175306677818298f, -2.244840621948242f, 1.9783128499984741f, 0.3803824186325073f, 0.04183865338563919f, 0.284446656703949f, -2.161471128463745f, -0.5391823053359985f, 0.3041086494922638f, -0.3503345549106598f, 1.171104907989502f, -0.8915714621543884f, 1.7047383785247803f, -1.7130810022354126f, 1.0918960571289062f, 1.9015947580337524f, -0.2993949353694916f, 0.2431759536266327f, -0.06963914632797241f, -1.9186826944351196f, -1.1858599185943604f, 1.3959892988204956f, 0.31790241599082947f, 1.0485270023345947f, -0.8889532089233398f, -0.1494847685098648f, 0.011521455831825733f, -0.3648742437362671f, -0.14563268423080444f, 1.1415576934814453f, 2.0192317962646484f, -1.1988294124603271f, -0.08969400823116302f, 1.01350998878479f, -0.148199200630188f, -0.20676261186599731f, -0.4151966869831085f, -0.5613080859184265f, -1.5219647884368896f, -0.6867799758911133f, -1.207437515258789f, 0.27414625883102417f, 1.5931971073150635f, 0.8239222168922424f, 0.17494884133338928f, 0.37811675667762756f, -0.5250972509384155f, 1.1867752075195312f, 0.44894057512283325f, 0.6020882725715637f, 0.27788910269737244f, 0.3914108872413635f, -0.669601321220398f, -0.08302459120750427f, 0.06917848438024521f, -0.263455331325531f, 0.7421362996101379f, 0.2899683713912964f, -0.4380805194377899f, -1.3862485885620117f, -0.7712535858154297f, -0.3168620467185974f, -1.6128871440887451f, -0.9478936195373535f, -0.37378421425819397f, -0.8723973631858826f, 0.43149101734161377f, -0.7728766202926636f, -0.8030177354812622f, 0.46446937322616577f, -0.17089581489562988f, 2.706796407699585f, 0.6624157428741455f, -0.654021143913269f, 0.7278003692626953f, 0.09257330745458603f, -0.1797974407672882f, 0.7003864645957947f, -1.250577688217163f, 0.9090378284454346f, -0.15779435634613037f, -0.43905171751976013f, -0.16366083920001984f, -1.2389750480651855f, -1.0557494163513184f, 1.2461082935333252f, -0.7245869040489197f, -1.044531226158142f, 0.5626670122146606f, 0.17291614413261414f, 1.9904685020446777f, -1.181625247001648f, 0.30392396450042725f, 2.225896120071411f, 0.30583375692367554f, -0.6297805309295654f, 0.7762312889099121f, -0.5106104016304016f, -0.5062925219535828f, 0.7492969632148743f, 1.0681610107421875f, -2.636631488800049f, 0.6954406499862671f, -0.32303762435913086f, 0.5969774723052979f, -0.6247759461402893f, 1.647339940071106f, -1.7078118324279785f, 0.09625459462404251f, -0.28645211458206177f, 1.0167992115020752f, -0.6579540371894836f, -0.3999125063419342f, -1.7973577976226807f, -0.9080927968025208f, 1.1592087745666504f, -1.467360258102417f, 1.7191014289855957f, -0.3673190176486969f, 2.3575830459594727f, -0.459722101688385f, -0.058640673756599426f, -1.2418878078460693f, -0.023634955286979675f, -2.0428922176361084f, 0.19053655862808228f, 0.5756992697715759f, -0.5177616477012634f, 0.9197074770927429f, 0.9861821532249451f, 0.9352383017539978f, 0.30303436517715454f, -0.5343928933143616f, -0.2886335253715515f, 0.9647671580314636f, 1.7285969257354736f, -0.8043005466461182f, -0.28731220960617065f, -1.2584627866744995f, 0.08387812972068787f, 0.9973886609077454f, -0.7661278247833252f, -0.3684079647064209f, -0.16042770445346832f, 0.731488823890686f, -0.4989534318447113f, 0.5183284878730774f, 0.6223585605621338f, 0.017574317753314972f, 1.3758805990219116f, -0.024098770692944527f, -0.11441737413406372f, -0.2840602993965149f, -0.10638472437858582f, 0.3422453701496124f, -0.21973253786563873f, -0.09440521895885468f, -0.514839768409729f, 0.5137941241264343f, 0.3945278227329254f, 0.5612776279449463f, -0.006535662803798914f, 0.8489260077476501f, 0.4735972583293915f, 0.5418957471847534f, 0.0022529142443090677f, -0.5435486435890198f, 0.18075302243232727f, -0.7696079015731812f, -1.0346392393112183f, 0.7314628958702087f, 1.6726516485214233f, 0.35546284914016724f, 0.30347952246665955f, 0.04806268960237503f, 0.5032230019569397f, -0.025565603747963905f, -1.8534711599349976f, 1.0129450559616089f, -0.07218152284622192f, 0.03116939589381218f, -0.755984902381897f, -0.7956190705299377f, -0.6868169903755188f, 1.9091075658798218f, -0.4942598342895508f, -0.06308790296316147f, 0.9503589868545532f, 0.6692175269126892f, 0.12500713765621185f, 0.5085462331771851f, -1.0646655559539795f, -0.4647725224494934f, -0.9152674674987793f, 0.7144214510917664f, 1.472544550895691f, -1.7723859548568726f, 0.0483609139919281f, 0.20343580842018127f, -0.6890958547592163f, -0.5412123203277588f, 2.0465071201324463f, -0.5884720087051392f, -0.8352398872375488f, 0.0502433180809021f, 0.4375612735748291f, 1.3833050727844238f, -1.3894257545471191f, 0.3897654116153717f, -1.5383917093276978f, 1.8197290897369385f, -0.5204964280128479f, 1.4799479246139526f, 0.4248008131980896f, -0.30355918407440186f, 0.07046826928853989f, -1.5127924680709839f, 1.3581115007400513f, 0.32402706146240234f, -1.0680902004241943f, 0.08959023654460907f, 3.1311562061309814f, -0.8544708490371704f, -0.8357695937156677f, 1.2985942363739014f, -0.7110929489135742f, -0.9653112292289734f, -1.340206503868103f, 0.03399703651666641f, -0.4458200931549072f, 0.151767760515213f, 1.6231015920639038f, -0.429679274559021f, -0.16159531474113464f, -1.0276970863342285f, -0.9918712973594666f, 0.29410406947135925f, -0.3595946729183197f, -0.13289135694503784f, -0.09766664355993271f, -1.9935065507888794f, 0.11608057469129562f, 0.9187806844711304f, 0.12344544380903244f, 2.8273682594299316f, 0.47407886385917664f, -1.9149128198623657f, -1.5384353399276733f, -0.6529980897903442f, 0.16414928436279297f, -0.19700200855731964f, 1.038715124130249f, -0.3908529281616211f, 1.0207839012145996f, -1.2268577814102173f, -2.0579991340637207f, 0.5987722277641296f, -0.13595744967460632f, 0.24731674790382385f, -2.150991439819336f, 1.2150318622589111f, -0.33008328080177307f, -0.5156939625740051f, 0.022917015478014946f, -0.6746217608451843f, 1.309282898902893f, -1.6657313108444214f, 0.32850465178489685f, -0.8491258025169373f, 0.9747982025146484f, -0.37322181463241577f, 0.6090918779373169f, -0.7589855194091797f, -0.6492360830307007f, -1.2674946784973145f, -0.382905513048172f, 0.8315308690071106f, 1.8437397480010986f, 0.021685972809791565f, 0.28762421011924744f, 0.39912644028663635f, 2.4587013721466064f, 0.4567311406135559f, 1.106606125831604f, 0.9169553518295288f, -0.23127765953540802f, -1.498667597770691f, 0.6427391171455383f, 0.18222902715206146f, -1.0757185220718384f, 1.345549464225769f, 0.32771772146224976f, 1.3408823013305664f, 1.2158557176589966f, 0.95885169506073f, 0.5136737823486328f, 0.5763065814971924f, 0.0778578370809555f, -1.061161994934082f, 2.042306900024414f, 0.6509259343147278f, -1.0072094202041626f, 0.35781362652778625f, -1.0799492597579956f, 0.04710597172379494f, 2.201401472091675f, 0.2908516526222229f, -0.08251824229955673f, 1.5195056200027466f, -0.3584267199039459f, -0.6102548837661743f, -1.5060930252075195f, -0.7732543349266052f, 1.0847911834716797f, 1.4872429370880127f, 0.6041548848152161f, -1.1257244348526f, 0.3761575222015381f, 1.959293246269226f, -0.7958163619041443f, 0.4539487659931183f, -2.5069937705993652f, -1.361000418663025f, 1.183761477470398f, -1.1673825979232788f, -0.25077250599861145f, 0.33264845609664917f, 1.1230510473251343f, 1.8761743307113647f, 0.273639976978302f, 0.2801147401332855f, 0.7281787991523743f, 0.02403142862021923f, 2.3937904834747314f, -1.3129955530166626f, -0.48624828457832336f, 0.4012764096260071f, 1.0767757892608643f, 1.1279785633087158f, 0.02586616761982441f, -0.14947263896465302f, 0.39506977796554565f, 1.9742354154586792f, -0.4404173791408539f, 2.3815414905548096f, -0.34589144587516785f, 2.694831609725952f, -0.8544124364852905f, -0.27781322598457336f, 0.7772916555404663f, -0.43923330307006836f, 1.1729096174240112f, 1.397361159324646f, -0.41255858540534973f, -0.4339471161365509f, 0.9574882388114929f, 1.254128336906433f, -1.0384563207626343f, 0.5251074433326721f, -0.8383287787437439f, 1.7227904796600342f, -1.1731188297271729f, 0.7272883057594299f, -1.7194626331329346f, 2.383774995803833f, -1.1649107933044434f, -0.20503655076026917f, -0.5375787019729614f, 1.2967820167541504f, -1.6138769388198853f, 0.7535606026649475f, -0.38888975977897644f, 2.5921998023986816f, 0.08543383330106735f, -2.0308878421783447f, 0.6461803913116455f, 1.217452883720398f, 0.03121299482882023f, 1.082465648651123f, -1.8998514413833618f, -2.7074427604675293f, -0.7867730259895325f, -0.6195639967918396f, 1.8281400203704834f, 0.4877544343471527f, 0.1366790533065796f, -0.5910199284553528f, 2.3618080615997314f, 1.2076122760772705f, -0.5487976670265198f, 1.5311497449874878f, 0.44362834095954895f, -2.4579336643218994f, 0.6496618390083313f, 0.02449978142976761f, 0.25982725620269775f, 0.7348682284355164f, -0.006029692944139242f, -0.7884382605552673f, 1.1371253728866577f, -1.7366209030151367f, 0.9614791870117188f, -1.3104760646820068f, -1.4218922853469849f, -0.3201002776622772f, 0.011662798002362251f, 0.7528783082962036f, -0.8849666118621826f, 0.8982152342796326f, -1.6744109392166138f, -1.1447242498397827f, 0.22346419095993042f, -0.9369489550590515f, -0.17544429004192352f, 0.8354768753051758f, -1.2516489028930664f, 0.26434803009033203f, 1.2949540615081787f, -0.5020677447319031f, 0.45930108428001404f, -1.3068112134933472f, 0.9302626848220825f, 1.0709624290466309f, 1.3384746313095093f, 0.27829068899154663f, -1.328766107559204f, -0.2891634404659271f, -2.518200635910034f, -2.533468723297119f, -0.022988365963101387f, -0.5563327670097351f, 2.0752601623535156f, -0.8403553366661072f, 0.9517558813095093f, -2.1648738384246826f, 0.9794597625732422f, 0.38047143816947937f, -1.9277540445327759f, 0.9197121858596802f, -1.6067678928375244f, -1.3559657335281372f, -0.25936615467071533f, 0.0861901044845581f, -0.37659966945648193f, 0.9756653904914856f, -0.8411983847618103f, -0.9621247053146362f, -0.5808306932449341f, 1.1524169445037842f, -0.5585207939147949f, -0.13208438456058502f, -1.0929938554763794f, -2.4860122203826904f, -0.038849566131830215f, 0.8021038770675659f, -0.18547378480434418f, 0.534229576587677f, -0.49810588359832764f, 0.8963930010795593f, -1.2691730260849f, 0.042280785739421844f, 0.9680984020233154f, -1.7608284950256348f, 1.0980932712554932f, 0.8579375743865967f, -0.9562888145446777f, -0.7236151099205017f, 0.2793406844139099f, -0.8662601113319397f, 0.8459339737892151f, 0.6116387844085693f, -0.8089803457260132f, -0.20030127465724945f, -2.4035587310791016f, 1.5633418560028076f, -0.6294617652893066f, 0.8042920231819153f, 0.6676139235496521f, -0.5181459784507751f, 0.9539962410926819f, 0.7256157398223877f, -0.5767995715141296f, -0.01296310406178236f, 1.34257972240448f, 0.8595607280731201f, 0.24403470754623413f, 1.9769524335861206f, 0.5327519178390503f, -0.8788067102432251f, -0.2609003186225891f, -0.1277228742837906f, -0.38701093196868896f, -0.10875839740037918f, 0.7135027647018433f, 0.037005677819252014f, 0.7367677688598633f, -1.073968529701233f, -0.8155962824821472f, 0.3688916862010956f, 0.5232675075531006f, -1.1358050107955933f, 2.01452898979187f, -0.5778390765190125f, -0.2187574803829193f, 0.35567042231559753f, 0.7510744333267212f, 0.2171924114227295f, 0.4019394814968109f, 0.9250243306159973f, -0.24931922554969788f, -0.3473069369792938f, 0.8122217059135437f, -0.6078834533691406f, -0.8096639513969421f, -0.4163765013217926f, -0.9129456877708435f, -1.0231975317001343f, 0.15973883867263794f, 1.5596561431884766f, 0.2012278288602829f, 0.9135115742683411f, -0.7519399523735046f, 2.7958858013153076f, 0.41675102710723877f, 0.0033115341793745756f, -1.398554801940918f, 1.5899420976638794f, -0.7806249856948853f, -0.7495994567871094f, 0.2814400792121887f, -0.08736804872751236f, -1.1191902160644531f, -0.4140253961086273f, 0.2933676540851593f, -0.016588633880019188f, 0.4729222357273102f, 1.0479241609573364f, 0.32106152176856995f, 1.162279486656189f, 1.5198776721954346f, 1.134521245956421f, 0.013782661408185959f, 1.5933806896209717f, 0.11106725037097931f, 0.5933053493499756f, 0.0023419519420713186f, 0.9275448322296143f, 0.1871725618839264f, 1.0549991130828857f, -0.04648401960730553f, 1.6230204105377197f, 0.9345911145210266f, 0.6804553866386414f, 2.2367258071899414f, -0.6858885884284973f, -1.308877944946289f, -1.208367109298706f, -1.4461109638214111f, -0.6192750930786133f, -0.40478718280792236f, 0.45292869210243225f, 0.40945965051651f, 0.9656304121017456f, 0.4625106155872345f, -1.4208135604858398f, 0.2507438659667969f, -0.5702037215232849f, -0.05394743010401726f, -0.7306843400001526f, -1.0346637964248657f, 0.7631427049636841f, 0.9210829138755798f, -0.6593214273452759f, 0.24809309840202332f, 0.7311660647392273f, 2.3288936614990234f, -1.06203031539917f, -0.5385692119598389f, 0.4127616584300995f, 0.3390391767024994f, 1.7734715938568115f, 1.655669927597046f, -0.08070362359285355f, -0.303115576505661f, 0.3240552544593811f, 1.0401465892791748f, 0.5915576815605164f, -1.4803826808929443f, -0.12040789425373077f, 0.16437974572181702f, 0.7806977033615112f, 1.228264331817627f, 0.4453698992729187f, -0.3023916184902191f, -0.7796450257301331f, 1.15239417552948f, -0.4312492907047272f, 0.4804547131061554f, -0.0037577631883323193f, -0.23722761869430542f, 0.1753881573677063f, -0.3416663408279419f, -0.4271427094936371f, 1.1991220712661743f, 0.8689031600952148f, -0.6073640584945679f, 0.11616495996713638f, 1.9142365455627441f, -1.1450425386428833f, 1.60905921459198f, -0.5139697194099426f, -0.3140712380409241f, -0.8430630564689636f, -1.2868314981460571f, 0.5520419478416443f, 2.584942102432251f, 0.4215601086616516f, 2.558887481689453f, -1.2483208179473877f, -0.9826495051383972f, 0.7373914122581482f, 1.4831291437149048f, 0.6203920245170593f, 0.33499792218208313f, -0.5796453952789307f, -0.24162966012954712f, 1.4769864082336426f, -1.1677722930908203f, 0.43188926577568054f, 0.7582098245620728f, 0.26907673478126526f, -0.16138136386871338f, 1.1750508546829224f, 0.5351220369338989f, 1.225508689880371f, 0.9085782766342163f, 0.16776901483535767f, -0.009964917786419392f, -0.9104782342910767f, 0.3974350690841675f, -0.5543625354766846f, 1.281534194946289f, 0.38322561979293823f, 1.3406338691711426f, -0.43185070157051086f, -1.897344708442688f, -2.652250289916992f, -0.43282368779182434f, 0.932184100151062f, 0.4349152445793152f, -0.7768413424491882f, -0.19172848761081696f, 1.1136181354522705f, -1.214506983757019f, 2.1966660022735596f, 0.41227126121520996f, -0.7043505311012268f, 0.09847358614206314f, 1.8926005363464355f, 0.6903649568557739f, 1.4352598190307617f, 1.0864993333816528f, -0.3989373445510864f, -2.0487279891967773f, -1.1373023986816406f, 0.9046337604522705f, 0.5279921293258667f, -0.5005130171775818f, 0.0378837063908577f, -1.0129538774490356f, 0.7843693494796753f, 1.3280545473098755f, -0.5138816833496094f, -0.04832516983151436f, -0.8721303939819336f, -1.3769567012786865f, -1.8487751483917236f, 0.30561649799346924f, -0.42448726296424866f, 0.6269863247871399f, -0.6438542604446411f, 3.0598578453063965f, 0.8688741326332092f, 0.7331935167312622f, 1.7552188634872437f, -0.24250611662864685f, 1.9880743026733398f, -1.1370364427566528f, -2.2499585151672363f, -0.5810695886611938f, 0.8816671967506409f, -2.0325005054473877f, -0.35224223136901855f, 0.1006089523434639f, 1.5255436897277832f, -1.6215003728866577f, 0.5298449397087097f, -1.3718810081481934f, -0.6298099756240845f, -0.65668123960495f, 0.4244276285171509f, -0.40122607350349426f, -0.4890124797821045f, 0.9571828246116638f, -0.8436367511749268f, -0.3680597245693207f, -0.47244831919670105f, -0.6993737816810608f, 0.4523862302303314f, -1.090578317642212f, 0.9411556720733643f, -1.931666612625122f, 0.1285097301006317f, -1.0134315490722656f, 0.7479439973831177f, 0.20112790167331696f, -0.8995627760887146f, -0.2906798720359802f, 0.1047126054763794f, -0.5943686366081238f, -1.727795124053955f, 1.0557186603546143f, -0.5037193298339844f, -1.5121967792510986f, -0.14455023407936096f, -0.7709440588951111f, 1.3850905895233154f, 0.06768468767404556f, -2.205592393875122f, -0.9360390901565552f, 1.6600853204727173f, -0.551665186882019f, -0.3103943169116974f, -0.6468674540519714f, -0.22681531310081482f, -0.3377947509288788f, 0.6926270723342896f, -0.5544835925102234f, -0.10487986356019974f, -0.12251695245504379f, -0.10804381966590881f, -1.4534186124801636f, -0.5115614533424377f, 0.9907002449035645f, 0.7906493544578552f, -0.34170281887054443f, -1.8583146333694458f, 0.6896669268608093f, 0.8880993127822876f, 1.2050648927688599f, -0.5951094031333923f, -0.3993092179298401f, -0.30013880133628845f, -0.014673005789518356f, 0.6156877279281616f, -0.9164139032363892f, -0.06423450261354446f, -0.33351314067840576f, 0.5679208636283875f, 0.5063413977622986f, 1.8924111127853394f, -0.7531763911247253f, 1.0589748620986938f, -0.010384922847151756f, 0.41770482063293457f, 0.16313597559928894f, -0.26724591851234436f, 0.1958005577325821f, 0.8279790282249451f, -0.7302268147468567f, 0.4149245321750641f, 2.0801644325256348f, 0.1493149846792221f, -0.4159306585788727f, 1.2188113927841187f, 0.8605815768241882f, 0.8988009691238403f, -0.4320070743560791f, -0.011584565043449402f, 0.1905190795660019f, 0.45351630449295044f, -0.2722189724445343f, 0.4377744197845459f, 0.6674123406410217f, -0.8114345669746399f, -1.1677125692367554f, 1.4608654975891113f, -1.6388529539108276f, 0.9854339957237244f, 0.01134670153260231f, 0.3140886127948761f, 1.081308126449585f, 0.1636349856853485f, -1.8501884937286377f, -0.6727942824363708f, -1.8205945491790771f, -0.16954289376735687f, -0.974631667137146f, -0.5593003630638123f, -0.2645290195941925f, 0.2448410987854004f, 0.4654839038848877f, 1.1221051216125488f, -1.2755296230316162f, 1.968849539756775f, 1.1931818723678589f, -0.9534568786621094f, 0.6892605423927307f, -0.03748741000890732f, -1.3567429780960083f, -0.930943489074707f, -0.4104326069355011f, 1.6298463344573975f, -0.3964575529098511f, -0.8092941045761108f, -0.8157675862312317f, -0.1790880262851715f, -0.7333499789237976f, 2.0713958740234375f, 0.26833969354629517f, -0.4909171462059021f, 0.0867118388414383f, -0.19213901460170746f, -0.020139964297413826f, 1.3420833349227905f, 0.8771296143531799f, 0.9378947615623474f, 0.023886969313025475f, 1.5275574922561646f, -0.08036366105079651f, -0.9117053151130676f, -1.1944957971572876f, 0.9197462201118469f, 0.28947049379348755f, 0.5755492448806763f, -1.0247280597686768f, 0.19418835639953613f, 1.5901795625686646f, -0.7810332179069519f, 0.18075843155384064f, 0.5682910680770874f, -1.3145383596420288f, -0.037264980375766754f, -1.1747896671295166f, -0.5241218209266663f, 0.8552600145339966f, 0.7151663303375244f, 0.854972243309021f, 0.6970289349555969f, -0.7869400978088379f, -0.22646142542362213f, 0.8945121765136719f, -0.07112658768892288f, 0.19750215113162994f, -0.08805552124977112f, 0.46580448746681213f, 0.745457112789154f, -0.6745474338531494f, -0.8672813177108765f, -2.0061123371124268f, 2.204427480697632f, -0.336713045835495f, -0.07664714008569717f, 0.6008157134056091f, 0.7986276745796204f, -1.3824840784072876f, -0.3528830111026764f, 0.9141533374786377f, 0.5920729637145996f, 0.7283052206039429f, 0.5198004245758057f, 0.40039631724357605f, -0.16163653135299683f, 0.3812867999076843f, 0.2818247973918915f, -0.259854257106781f, -0.17636136710643768f, 1.2899055480957031f, -1.809929609298706f, 0.04499081149697304f, -0.5114196538925171f, 1.6922872066497803f, -0.5404288172721863f, -0.06902796775102615f, -0.900188148021698f, 0.16361606121063232f, -0.5697844624519348f, -0.17199715971946716f, 1.9494951963424683f, -1.5725066661834717f, -0.11967476457357407f, 0.6768812537193298f, -2.550161600112915f, 1.3886760473251343f, 2.071373224258423f, -0.4157007336616516f, 1.7397890090942383f, -1.5081530809402466f, -0.9471526145935059f, 0.4156656265258789f, -0.7409192323684692f, -2.0307180881500244f, 0.7459039092063904f, -0.23346112668514252f, -1.7168304920196533f, -1.5648266077041626f, 1.1608803272247314f, -1.7281652688980103f, 0.6240199208259583f, 0.19563332200050354f, 0.9567276835441589f, 0.9916862845420837f, -0.7535449862480164f, 1.3851630687713623f, 0.8076578974723816f, 0.4444909691810608f, 0.12085723131895065f, 0.794389545917511f, -0.2691969871520996f, 1.6320013999938965f, 1.4558833837509155f, 0.8513883352279663f, 1.305290937423706f, 0.8631715178489685f, -0.5379660725593567f, 0.5391438603401184f, -0.18788452446460724f, -1.6509469747543335f, 0.6196591854095459f, -1.1270289421081543f, -0.08517079800367355f, 1.0406969785690308f, -0.5241701006889343f, -0.5131880044937134f, 0.4295358955860138f, 1.8150509595870972f, -1.150861382484436f, -0.30584630370140076f, 0.8076725602149963f, 0.7956660985946655f, 0.2235465794801712f, -0.05132210999727249f, 0.7380313277244568f, 1.5020791292190552f, 0.8205595016479492f, -0.056694693863391876f, -0.17149198055267334f, 0.9732901453971863f, -0.9835246801376343f, 0.1294081211090088f, -1.1072022914886475f, -0.1295740306377411f, 1.005579948425293f, -1.3748044967651367f, 1.197999119758606f, -2.2044677734375f, -1.1458145380020142f, -1.4537255764007568f, -0.2384955883026123f, -1.6215006113052368f, -0.3137211501598358f, -0.7217759490013123f, -1.6709321737289429f, 0.3768741488456726f, -1.8265146017074585f, 0.48899734020233154f, -0.5580688714981079f, 0.37294480204582214f, -0.6878536939620972f, -1.4576401710510254f, -0.6108043789863586f, -0.6761788725852966f, -1.9144877195358276f, 0.9340372085571289f, -0.4341937303543091f, 1.4776188135147095f, 0.7693286538124084f, 2.331334352493286f, -0.6489352583885193f, 0.4903559982776642f, -0.2804304361343384f, 0.941003680229187f, -0.11338216066360474f, -0.8615462183952332f, 0.13828350603580475f, 0.013346405699849129f, 0.33505237102508545f, -0.24309676885604858f, -0.6505504250526428f, -0.33003056049346924f, 1.3748613595962524f, -1.033202886581421f, 0.05953902378678322f, -0.2076457291841507f, -0.9487320184707642f, -0.2869058847427368f, -0.902455747127533f, -1.5298161506652832f, -1.2186866998672485f, 1.1458920240402222f, 0.20151416957378387f, -0.4259151816368103f, -0.1506912112236023f, -2.106266498565674f, -0.6301612257957458f, 1.2424033880233765f, -0.025852564722299576f, -0.540851891040802f, -0.3480299413204193f, -0.7818117141723633f, 0.5524802803993225f, 0.8857057690620422f, -1.3390332460403442f, -0.40026965737342834f, 0.7468986511230469f, -0.3904872536659241f, 0.7475174069404602f, -0.4455646276473999f, 1.0798193216323853f, 2.6205930709838867f, 0.6864228844642639f, 1.5354714393615723f, -0.1950068324804306f, -1.4005751609802246f, 0.4377126097679138f, -0.05925627797842026f, -0.552335798740387f, -0.7325940728187561f, 0.5038686990737915f, -0.18185997009277344f, -0.7621927261352539f, -0.8477222323417664f, 0.6641071438789368f, 2.3913745880126953f, 0.4669826030731201f, 0.7512773275375366f, -0.3845667541027069f, -0.06630031019449234f, 0.6945987939834595f, -0.7470712065696716f, 0.2076980471611023f, -0.12689319252967834f, -0.5187065005302429f, 0.8009016513824463f, -0.7448427081108093f, 1.9110214710235596f, 0.3227183520793915f, -0.5433307886123657f, -2.0223824977874756f, 0.7815577387809753f, 1.8362489938735962f, -0.47440794110298157f, -0.38593149185180664f, 0.36727604269981384f, 0.344801127910614f, 0.5554131865501404f, 0.35217124223709106f, 0.7673863768577576f, 0.08270980417728424f, 0.3816434144973755f, -1.0023962259292603f, -0.22075708210468292f, -0.6525165438652039f, -2.485137939453125f, 0.614276111125946f, 0.9891844391822815f, -1.6560537815093994f, 0.17567120492458344f, 0.8857656121253967f, 0.022398322820663452f, -0.24084077775478363f, 1.4238216876983643f, 0.8160249590873718f, 0.2655353844165802f, 0.744394838809967f, -0.17409075796604156f, -0.9177995324134827f, -2.1325347423553467f, -0.4131704568862915f, -0.4764634966850281f, 0.5031798481941223f, 0.27043330669403076f, -0.04328599572181702f, 1.035592794418335f, 1.341705560684204f, -0.11890244483947754f, -0.07687754184007645f, 0.5947466492652893f, 0.5723721385002136f, 2.663830518722534f, 0.7388755679130554f, -0.46826601028442383f, -1.59627366065979f, -1.6671663522720337f, 0.3388274610042572f, 0.3750116229057312f, -1.3291982412338257f, 0.5636889338493347f, 0.8051766753196716f, 0.7449150681495667f, -0.1739114224910736f, 1.1078135967254639f, 0.5147720575332642f, 0.8934884667396545f, -1.5113967657089233f, -0.8514725565910339f, 0.3001462519168854f, 0.17312221229076385f, 0.08587908744812012f, 0.10766935348510742f, 1.0629868507385254f, 1.841042399406433f, -0.5686787366867065f, -1.44584059715271f, -0.2561878263950348f, 1.1728384494781494f, -0.428275465965271f, -0.2985764443874359f, 0.14890146255493164f, 1.661197304725647f, -2.622263193130493f, -0.43193310499191284f, -0.3783835768699646f, 0.23239530622959137f, -0.7321792840957642f, -0.40373992919921875f, 1.2062691450119019f, -0.6938396096229553f, -1.5164529085159302f, 2.0242106914520264f, -0.8075132966041565f, 0.09412498027086258f, 0.27298247814178467f, 0.5441054701805115f, -2.4758286476135254f, 0.21060240268707275f, 0.2536042630672455f, -0.7318422794342041f, 0.05695872753858566f, 0.15881912410259247f, 0.6510986685752869f, 0.020065616816282272f, 0.8478670120239258f, 2.0752015113830566f, -0.38090255856513977f, -0.9460003972053528f, -0.4590383470058441f, -0.6942368745803833f, 0.30009862780570984f, -1.389849305152893f, 1.2990328073501587f, -1.8040062189102173f, 1.837660551071167f, -0.7219073176383972f, -0.3056040108203888f, 0.9780712127685547f, 0.9052547812461853f, -0.901805579662323f, -0.2280527949333191f, -0.948022723197937f, -0.15365807712078094f, 1.032126545906067f, 0.17955327033996582f, -0.2721782922744751f, 0.15856090188026428f, 1.1139295101165771f, -0.5713488459587097f, -0.9870969653129578f, -1.6782877445220947f, 1.8284801244735718f, -0.4982566237449646f, 0.47082117199897766f, -0.591300904750824f, -0.41513174772262573f, 0.5321887731552124f, 1.317676305770874f, -0.7898051142692566f, -0.5088178515434265f, 0.23661386966705322f, 1.2172428369522095f, 0.2148703932762146f, 1.2481820583343506f, 2.4610743522644043f, 0.14153295755386353f, 1.837676763534546f, -0.2276618480682373f, 0.2129386067390442f, 0.8061029314994812f, 0.7021245956420898f, -0.6485925316810608f, -0.47295162081718445f, 2.2190706729888916f, -1.868390679359436f, -1.3732751607894897f, -1.013265609741211f, 1.365964651107788f, 1.1893959045410156f, -2.7800850868225098f, -1.0310310125350952f, -0.702293336391449f, 0.8099242448806763f, 0.8948581218719482f, -0.40090087056159973f, -0.08886078000068665f, -0.9508647918701172f, 0.28014296293258667f, 0.06467004120349884f, -2.057403087615967f, 1.1129963397979736f, -0.7580298185348511f, 0.0977560505270958f, 0.8891482949256897f, 0.3460270166397095f, -0.11394428461790085f, 1.4566701650619507f, -1.1309462785720825f, 1.2330803871154785f, 0.8789023160934448f, -2.017490863800049f, 0.856982946395874f, 0.04652906581759453f, -0.3498983681201935f, -0.8770002722740173f, -0.38342466950416565f, -0.05906866118311882f, 0.7605865001678467f, -0.9748004078865051f, 0.9013839960098267f, -0.43347686529159546f, -1.508413314819336f, 2.2633509635925293f, 0.88945072889328f, 0.7355136275291443f, -0.6224141120910645f, 0.2808963656425476f, 0.1529616415500641f, 0.09397149085998535f, 0.8745917677879333f, -1.226547122001648f, 1.3209213018417358f, 1.0341976881027222f, -0.49460795521736145f, -3.3865373134613037f, 2.0950191020965576f, 0.2728019058704376f, -0.030417079105973244f, 0.08612233400344849f, -1.7298319339752197f, 0.08109258860349655f, 1.2689827680587769f, -0.6729629635810852f, 0.8545035123825073f, -1.1793856620788574f, -0.9540548324584961f, -0.969673752784729f, 0.3141362965106964f, 1.2000207901000977f, -0.1831716001033783f, -0.18045181035995483f, -0.1034746766090393f, -0.10869846493005753f, 0.5782245993614197f, 1.7499538660049438f, 0.002034955658018589f, 0.36500951647758484f, -0.8946718573570251f, -0.219259113073349f, -0.47011202573776245f, -0.9500816464424133f, -0.6808398962020874f, -0.9808987379074097f, -0.7519583702087402f, -1.0324863195419312f, -1.8080289363861084f, 0.21794334053993225f, 0.6857624053955078f, 0.43733805418014526f, -1.9561761617660522f, 1.2915894985198975f, -1.6977379322052002f, -1.2025177478790283f, -1.646543025970459f, -1.3370387554168701f, 1.169712781906128f, -0.5579738020896912f, 0.38680657744407654f, -0.4910755455493927f, -0.32521501183509827f, 0.13093096017837524f, 0.624261200428009f, 0.6792762279510498f, 0.3959449231624603f, 0.66949063539505f, -0.19622361660003662f, -0.32004305720329285f, 0.20433473587036133f, -1.9621531963348389f, -0.5470468401908875f, 1.5721423625946045f, 1.0094727277755737f, 0.4029425382614136f, 1.6145533323287964f, 0.4475519359111786f, 0.5499113202095032f, 0.21999366581439972f, -0.019641423597931862f, 0.20199595391750336f, -0.305207222700119f, -0.11879883706569672f, 0.8017854690551758f, -0.3870505392551422f, 1.560797929763794f, 0.04964430257678032f, -0.33324524760246277f, 0.7588040828704834f, -1.5863133668899536f, -0.6730678081512451f, -1.2264569997787476f, 1.3401838541030884f, -1.162084698677063f, 1.4780147075653076f, 1.5603762865066528f, -0.3231433033943176f, 2.119018793106079f, 0.7877871990203857f, -1.9625552892684937f, -0.526032567024231f, 2.639968156814575f, 1.171855092048645f, 1.2722933292388916f, 0.4042589068412781f, -1.5217982530593872f, -0.5133728981018066f, -1.3806458711624146f, -0.5332193970680237f, 1.0821458101272583f, 0.6407410502433777f, -1.729612946510315f, -2.14782977104187f, 0.4973359704017639f, -1.268904685974121f, -0.24232769012451172f, -1.0745846033096313f, -0.6819858551025391f, 1.0070197582244873f, -0.8279359936714172f, 2.587069511413574f, -0.7386665940284729f, 0.3296529948711395f, 2.8114237785339355f, -0.879487931728363f, -0.33060166239738464f, 0.6457027792930603f, 0.17840361595153809f, -1.1289931535720825f, -0.3602311313152313f, -2.383305549621582f, 1.3506853580474854f, -0.4065081775188446f, -0.30150434374809265f, -1.5348396301269531f, 1.6021355390548706f, 0.6893559694290161f, 0.033526014536619186f, -0.2977294325828552f, -1.1700760126113892f, 1.5237687826156616f, 1.2984881401062012f, 0.6588833928108215f, -0.8222662210464478f, -0.14632517099380493f, 0.7788861393928528f, 0.6403340101242065f, 0.8075235486030579f, 0.5731524229049683f, 1.3903805017471313f, -0.23513318598270416f, -0.5386273264884949f, -0.8155969977378845f, -0.1509992778301239f, -0.5020706653594971f, 0.8036778569221497f, -1.1333380937576294f, -0.7435185313224792f, -0.7013434171676636f, -0.42178988456726074f, -1.5332636833190918f, -1.9342981576919556f, 0.3510490953922272f, -0.6404718160629272f, 0.019135747104883194f, -0.7149955034255981f, -0.7025302648544312f, 0.7700265645980835f, -0.7571792006492615f, -1.0642281770706177f, 1.9611531496047974f, -0.4390997588634491f, 0.8920031785964966f, 1.8720558881759644f, 0.24501417577266693f, 0.7017510533332825f, 0.9099079370498657f, -0.7323962450027466f, -0.6663155555725098f, -0.3277064561843872f, -1.439155101776123f, -1.2797472476959229f, -0.19622953236103058f, 0.19903564453125f, -0.48808231949806213f, -0.6568363904953003f, -0.4720822274684906f, 0.35971710085868835f, 0.5307507514953613f, 1.0079243183135986f, 1.6095112562179565f, 1.6016820669174194f, 0.020138248801231384f, 0.573236346244812f, 0.37203314900398254f, 0.22609540820121765f, -0.754463255405426f, -0.3379971385002136f, 0.41659975051879883f, 0.22890278697013855f, -0.5443984270095825f, 2.050001859664917f, 0.06459449976682663f, 1.448185920715332f, -0.20686893165111542f, 1.5211719274520874f, -1.2341058254241943f, -0.7871248126029968f, -0.7268313765525818f, -0.26047298312187195f, -0.1815493106842041f, 0.40858983993530273f, 0.21912771463394165f, -0.45708730816841125f, -0.651996910572052f, -0.32107922434806824f, 0.20424777269363403f, -0.8860291838645935f, 1.0496968030929565f, 0.5568186640739441f, 0.13751773536205292f, -0.15331187844276428f, 1.2169883251190186f, 1.7785981893539429f, -0.709164023399353f, -0.9840890765190125f, -0.17089858651161194f, 0.2695762813091278f, 0.3193606436252594f, -0.6905565857887268f, 0.3500501811504364f, -0.32124459743499756f, -0.25051894783973694f, 2.1708250045776367f, -1.229917049407959f, 0.3085583448410034f, -0.4612124264240265f, -1.0031081438064575f, -3.3694987297058105f, 0.7832650542259216f, -0.34354400634765625f, 0.34807446599006653f, -1.7184555530548096f, -0.8929248452186584f, -0.7456709742546082f, 0.1508672833442688f, -0.7863383889198303f, -0.350765198469162f, -0.04678618907928467f, -0.6908267736434937f, -0.38857290148735046f, 0.04591083899140358f, -0.3794901967048645f, 1.5770217180252075f, -0.042833227664232254f, 1.353956699371338f, -1.124684453010559f, -0.17863909900188446f, 0.20412735641002655f, 0.9273927807807922f, -0.32438376545906067f, 0.6851024031639099f, 1.0722298622131348f, -1.000272512435913f, -0.8011578917503357f, -1.122711181640625f, -0.7824243903160095f, -0.5122944712638855f, -0.8242338299751282f, -1.0234198570251465f, -1.3928337097167969f, -0.4097520112991333f, 0.8509801030158997f, 0.8881950378417969f, 0.5827952027320862f, 0.785405158996582f, 1.5136889219284058f, 1.251064419746399f, -1.9372137784957886f, -1.2051048278808594f, 1.1982215642929077f, -1.376950740814209f, 1.0670830011367798f, 0.507300615310669f, 0.4764442443847656f, 0.9100666046142578f, -1.501094937324524f, -0.11335238814353943f, -1.2001420259475708f, -0.7933986186981201f, -2.7309763431549072f, -0.014749551191926003f, -0.04001638665795326f, -0.11494863778352737f, 1.458335041999817f, 0.4836253225803375f, 1.7024855613708496f, -0.9683796763420105f, 1.086437702178955f, 1.8661012649536133f, 1.4962280988693237f, 1.022226333618164f, 1.5485637187957764f, -0.08855976164340973f, -0.88787841796875f, -0.6835475564002991f, -0.015076662413775921f, -0.030384592711925507f, -0.8306879997253418f, -1.7835168838500977f, -1.239241361618042f, 0.9374039173126221f, -0.12884187698364258f, 0.9388713836669922f, 0.2919924259185791f, 1.7478625774383545f, -0.8145643472671509f, 0.9247024655342102f, -1.5529402494430542f, 0.13943159580230713f, 1.0945836305618286f, 1.0349615812301636f, 0.03250877186655998f, 0.9967503547668457f, 0.6327362060546875f, 0.7831454873085022f, 0.5098334550857544f, 1.5633010864257812f, -0.46283742785453796f, 0.1497601568698883f, 0.6901775598526001f, -0.38219600915908813f, 0.4878943860530853f, 0.04518410563468933f, 1.9969353675842285f, -0.6955253481864929f, 0.5120656490325928f, -0.9795308709144592f, 2.3521711826324463f, 0.41813287138938904f, -0.1308819055557251f, -0.4394487142562866f, 1.6976182460784912f, 0.813541829586029f, -0.6087417602539062f, 0.9832689762115479f, 2.428079843521118f, 1.1440294981002808f, -0.101145900785923f, 0.09917446225881577f, 0.3814586102962494f, 1.6723263263702393f, -0.4240947961807251f, 1.8199645280838013f, -0.39397287368774414f, -0.4608776867389679f, 0.51866215467453f, 0.7784189581871033f, -1.5908970832824707f, 1.9877948760986328f, 0.02497357502579689f, -0.43002068996429443f, 1.0171878337860107f, -0.548503041267395f, 1.6083859205245972f, 0.5863268971443176f, 0.15758942067623138f, -0.9828112125396729f, 0.18478159606456757f, 1.4150960445404053f, 0.4230518639087677f, 0.5434484481811523f, -1.2116377353668213f, 0.1757367104291916f, 0.25499454140663147f, 0.8070029616355896f, -0.11587464064359665f, 0.45107510685920715f, -0.3600947856903076f, -1.0252388715744019f, 0.37429946660995483f, -0.3791492283344269f, 2.650177001953125f, -1.6031533479690552f, -0.47765374183654785f, -0.5845633149147034f, -0.21724168956279755f, -0.5731971859931946f, 0.37175968289375305f, -0.4610559642314911f, 2.174776315689087f, 0.021914366632699966f, 0.1974239945411682f, -0.7603867650032043f, -0.693874180316925f, -1.0808472633361816f, -0.06485684216022491f, -0.010432514362037182f, 1.2084461450576782f, -0.9010456800460815f, -1.3052830696105957f, -0.12543383240699768f, 1.2808754444122314f, 0.6577393412590027f, 0.46201997995376587f, -0.5072460770606995f, -0.9180205464363098f, 0.0884491354227066f, -1.0534428358078003f, 0.2707360088825226f, -0.25277644395828247f, 0.016658902168273926f, -0.2014150619506836f, 0.05029870569705963f, 1.4853851795196533f, 0.8810616731643677f, 0.9759232997894287f, 0.807918131351471f, -1.3484669923782349f, 1.1456364393234253f, 0.31596988439559937f, -0.4552096426486969f, -0.18135374784469604f, -0.3125359117984772f, 2.018014669418335f, -0.44383805990219116f, -0.7552081942558289f, -1.7850168943405151f, -0.07548937201499939f, -0.5207358598709106f, -1.048349142074585f, 0.8424208760261536f, -0.9976255297660828f, 0.7335740327835083f, 1.2276335954666138f, -0.7577790021896362f, 1.622873306274414f, 1.292340874671936f, -0.26265472173690796f, -0.8315040469169617f, 1.0420149564743042f, 1.154740810394287f, 0.31027355790138245f, -0.6142987608909607f, 1.667606234550476f, 0.014784318394958973f, -1.3251757621765137f, -0.8346104025840759f, -0.2093493789434433f, 0.2113834023475647f, -0.15933676064014435f, 1.438138484954834f, -0.8965448141098022f, 0.4655975103378296f, -0.7802034020423889f, -0.7512221336364746f, -0.20733071863651276f, 0.41877850890159607f, 1.7955039739608765f, -0.6994057297706604f, -0.4367755055427551f, 0.34429505467414856f, -0.5315167307853699f, -1.7878962755203247f, 0.5759621858596802f, -0.37107211351394653f, -0.02148612029850483f, -0.3246121108531952f, 0.7541995644569397f, -1.7302361726760864f, -0.6404775977134705f, -0.024382900446653366f, -0.7490391731262207f, -2.1543679237365723f, -0.15692853927612305f, 0.09731115400791168f, 0.2538682818412781f, 1.5049229860305786f, 1.313053011894226f, -0.7395356297492981f, -1.813647985458374f, -0.2881675958633423f, -0.4023030996322632f, 0.7287482619285583f, -0.49943476915359497f, 0.7028453350067139f, 0.34216001629829407f, -1.3413481712341309f, 0.41704463958740234f, -0.8712550401687622f, -0.16223306953907013f, -0.9481205940246582f, -0.6534963846206665f, -0.3585118353366852f, -0.817564845085144f, 0.3736099898815155f, 0.9122175574302673f, 0.6781302690505981f, 0.055852312594652176f, -1.3394618034362793f, -0.30016884207725525f, 0.15999306738376617f, -0.6769115328788757f, 0.7504494786262512f, 0.216787651181221f, 0.23502039909362793f, 1.8141969442367554f, -1.5329066514968872f, -0.3652367889881134f, -1.5126714706420898f, 0.7930607199668884f, -1.0631688833236694f, 0.9984860420227051f, 1.037604570388794f, -0.04009038954973221f, -1.5908479690551758f, 0.30711954832077026f, 1.1175891160964966f, -1.345697283744812f, 1.0572247505187988f, 1.3808448314666748f, -0.366502046585083f, -0.08894097805023193f, -0.662988007068634f, -0.291555255651474f, 0.10108345001935959f, 1.0123193264007568f, -0.815822958946228f, -0.18586060404777527f, -0.7641056776046753f, -0.3749435842037201f, -0.8628830313682556f, -0.8098255395889282f, -0.21431559324264526f, -1.1151552200317383f, 0.14144384860992432f, -0.17559614777565002f, -0.7265163660049438f, -0.5217191576957703f, 0.37568187713623047f, 0.5935606956481934f, 0.20674307644367218f, 1.4827877283096313f, 0.9587778449058533f, -0.3417870104312897f, -1.0058722496032715f, -1.0774322748184204f, -2.2288825511932373f, 0.8113047480583191f, 0.13623955845832825f, -0.30091696977615356f, -0.2654571533203125f, -1.8403681516647339f, 0.1881687343120575f, -1.4522165060043335f, 1.4667081832885742f, 0.9349684119224548f, 0.25753054022789f, 0.9158172607421875f, -0.5319824814796448f, 0.6224501132965088f, 0.12310407310724258f, -0.17781615257263184f, 2.9330546855926514f, 0.4627670347690582f, -0.6264359354972839f, -1.3771589994430542f, -1.6635388135910034f, -0.2668311595916748f, -1.2189315557479858f, 0.14258776605129242f, -0.31570670008659363f, 1.423683524131775f, 1.0125263929367065f, -0.7147458791732788f, 0.3446122407913208f, 0.8802022337913513f, -0.7164220213890076f, -1.3230516910552979f, 1.0481867790222168f, -1.7053356170654297f, 0.600459098815918f, 1.8295810222625732f, -0.340763121843338f, 0.34682390093803406f, -0.1821630895137787f, -0.12917080521583557f, 0.7775542140007019f, 0.3030397295951843f, -0.8394461274147034f, 0.4900873601436615f, 0.9002130031585693f, -1.4860044717788696f, 0.9374179840087891f, -0.5453174114227295f, 0.485347181558609f, 0.6994611024856567f, -0.28342336416244507f, -0.11592065542936325f, 0.22056539356708527f, -0.8155755400657654f, 1.1422392129898071f, 0.16099725663661957f, 1.1248847246170044f, 0.09503551572561264f, 0.6680463552474976f, 0.22494570910930634f, -0.8516902327537537f, 0.4524131119251251f, -0.7356429100036621f, 1.461809754371643f, 0.056686531752347946f, -0.6240271329879761f, -0.3087240159511566f, 0.18612052500247955f, 0.026730623096227646f, -0.8790956139564514f, 1.3807705640792847f, -0.6990379691123962f, -1.2263665199279785f, -1.9690755605697632f, 2.3512303829193115f, 0.35396817326545715f, -0.10606183111667633f, -2.408507823944092f, -1.2640177011489868f, -0.4252816140651703f, 0.24079595506191254f, -0.24080689251422882f, -0.6830735802650452f, 0.28334593772888184f, -0.7549530863761902f, 0.685922384262085f, -0.28879889845848083f, 0.16147172451019287f, -0.18717071413993835f, 0.7061312794685364f, 0.12637053430080414f, -0.2261367291212082f, -0.40762385725975037f, -0.6097135543823242f, -1.347667932510376f, 0.8310710787773132f, 0.23480524122714996f, 0.23941050469875336f, -0.15233780443668365f, 1.35030198097229f, 0.3163374066352844f, -0.686985969543457f, 0.31536293029785156f, -1.2173928022384644f, -0.23235148191452026f, -0.11819366365671158f, -0.7065277099609375f, 0.8570377826690674f, -0.6065717935562134f, -2.5028295516967773f, -0.5181497931480408f, -0.3955882489681244f, -0.007207863032817841f, -1.8654048442840576f, -1.2592604160308838f, 0.44495517015457153f, 1.5878770351409912f, -1.009257197380066f, 2.5517168045043945f, 0.27151748538017273f, -0.45277684926986694f, 0.7783017754554749f, 0.5624421238899231f, -1.7339200973510742f, 0.1151348203420639f, 0.5969645380973816f, 1.4009592533111572f, 1.26650869846344f, 0.5665708780288696f, 1.4182316064834595f, 0.8785027265548706f, 1.713185429573059f, 0.5062318444252014f, 0.9417456984519958f, 1.1770522594451904f, -0.8855880498886108f, -0.897435188293457f, -0.4500139355659485f, -1.2818336486816406f, -0.09055130183696747f, 1.568543553352356f, 0.6288840770721436f, -0.13983508944511414f, -0.8341673016548157f, -1.0642942190170288f, -0.4966285824775696f, -0.8965734243392944f, 0.2338687777519226f, 0.13322870433330536f, -1.087693452835083f, -0.6877674460411072f, -0.034093379974365234f, -1.2961610555648804f, 1.018326997756958f, 2.094313621520996f, 0.22312399744987488f, 0.04033441096544266f, -0.11042066663503647f, 1.209588646888733f, 0.8461918830871582f, 1.0998334884643555f, -2.8670525550842285f, 0.43270260095596313f, 2.3624584674835205f, -0.9460180401802063f, 0.40504685044288635f, 0.21921862661838531f, -1.154799461364746f, 1.519834041595459f, -1.8380980491638184f, 1.218536376953125f, -0.1873958259820938f, 1.0788236856460571f, 0.14058129489421844f, -0.9915911555290222f, -1.5487273931503296f, -1.3351367712020874f, -1.668448805809021f, -1.84798002243042f, 0.02599930204451084f, 0.7676464319229126f, -1.9874215126037598f, 0.9982309937477112f, 0.003224122803658247f, 0.07818755507469177f, -1.0705227851867676f, -1.0210168361663818f, 1.7321865558624268f, 0.5420076847076416f, 0.3053724765777588f, 1.1364459991455078f, 0.6844477653503418f, 0.2691081464290619f, 0.8210340738296509f, 0.27511492371559143f, -0.38933637738227844f, -0.20226091146469116f, -0.19409379363059998f, -0.07234596461057663f, -0.7745715975761414f, 0.2601335644721985f, 1.2001798152923584f, -1.078745722770691f, 1.2962092161178589f, -0.18015773594379425f, -1.9301296472549438f, 0.28462305665016174f, 0.7962566614151001f, -0.26019519567489624f, 0.30697566270828247f, -0.7991265654563904f, 0.6080788373947144f, -0.8028495907783508f, -0.46709370613098145f, 0.5347719192504883f, 1.8560413122177124f, -0.11687781661748886f, 0.6693155169487f, -0.06095214933156967f, -0.1428632140159607f, -1.3090907335281372f, 0.03258459270000458f, 1.4419121742248535f, -0.6330651044845581f, -0.03063541278243065f, 1.5843851566314697f, -0.467591792345047f, -0.03230065852403641f, 0.7200000286102295f, -0.988913357257843f, -1.0607333183288574f, 0.7062268257141113f, -0.2287754863500595f, -0.030369358137249947f, 0.7900131344795227f, -0.22298280894756317f, 1.508110761642456f, 0.599722146987915f, -0.33899757266044617f, 0.1548950970172882f, -0.2794382572174072f, 0.8708680272102356f, 0.7626234889030457f, 0.6961768865585327f, 0.44085901975631714f, -0.4885202944278717f, 0.7180350422859192f, 1.8486416339874268f, -2.431525230407715f, -0.4701055586338043f, 0.4380730092525482f, 0.7702670693397522f, -0.14850954711437225f, 0.24941392242908478f, 0.539443850517273f, 0.058982912451028824f, 0.28753769397735596f, -0.6253249645233154f, 1.5168472528457642f, -0.610085129737854f, -1.8375158309936523f, 1.8140860795974731f, 0.9514070153236389f, -1.2033350467681885f, -0.5142884254455566f, -0.8533352017402649f, -0.5986783504486084f, -0.2986593544483185f, -0.7190134525299072f, -1.4356969594955444f, 0.6340908408164978f, -0.5407924652099609f, 1.3264530897140503f, 0.29634353518486023f, 0.17156629264354706f, 0.8127211332321167f, -0.636906087398529f, -1.1077786684036255f, 1.5005162954330444f, 0.5292592644691467f, -1.5582870244979858f, -0.22035345435142517f, -0.8878109455108643f, -0.23165416717529297f, -0.48117324709892273f, -0.8784040212631226f, 0.039074741303920746f, 0.03500647097826004f, -1.3603416681289673f, -0.3714776337146759f, -0.040006835013628006f, 0.5846149921417236f, 0.5806360840797424f, 0.7450182437896729f, 1.4870532751083374f, -0.051454752683639526f, 0.2527291476726532f, 0.7950483560562134f, 1.2641831636428833f, -0.0403965562582016f, -0.19124414026737213f, -0.4378063976764679f, -0.009368259459733963f, 0.25679612159729004f, -0.003075298387557268f, -1.1646342277526855f, -0.46010419726371765f, 0.18153296411037445f, 0.18499712646007538f, -0.23162956535816193f, 2.05483078956604f, 0.4368106722831726f, 1.1093194484710693f, 1.684390902519226f, -0.11415659636259079f, 0.37000924348831177f, 0.2512214779853821f, 1.1637510061264038f, -0.876596987247467f, -0.6737048625946045f, -0.652714192867279f, -0.753913164138794f, -0.10021805018186569f, -2.3608896732330322f, 2.2464849948883057f, -1.1070212125778198f, 1.026766061782837f, 0.2865336239337921f, -0.10950178653001785f, 2.380542039871216f, -0.41917189955711365f, 0.6831162571907043f, 1.7866971492767334f, -0.022699790075421333f, 0.7491136193275452f, -1.1757766008377075f, 0.2672518789768219f, -0.22947388887405396f, 0.5409282445907593f, 0.6939815282821655f, 1.8563321828842163f}; diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/data/layer.h b/sw/vme/apps/tew32/fp32-matmul-vle-vtse/data/layer.h deleted file mode 100644 index 54e4009f2..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/data/layer.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2024 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include - -/** - * @brief VME GEMM layer configuration. - * - * Describes a C = A * B problem with: - * A : M × K (stored row-major; kernel receives pre-transposed At : K × M) - * B : K × N (stored row-major) - * C : M × N (fp32 accumulator, tew=32) - * - * TM / TN are the micro-tile sizes used by the 2×2 blocked VME kernel. - * Typical values: TM = TN = 16. - */ -typedef struct { - uint32_t M; /**< Rows of A / rows of C */ - uint32_t N; /**< Columns of B / columns of C */ - uint32_t K; /**< Inner (reduction) dimension */ - uint32_t TM; /**< Micro-tile height (M direction) */ - uint32_t TN; /**< Micro-tile width (N direction) */ -} vme_gemm_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/kernel/gemm.c b/sw/vme/apps/tew32/fp32-matmul-vle-vtse/kernel/gemm.c deleted file mode 100644 index 6814d27d3..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/kernel/gemm.c +++ /dev/null @@ -1,551 +0,0 @@ - -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include "gemm.h" - -enum { - GEMM_TSS_MT0_ROW0 = 0u << 27, - GEMM_TSS_MT4_ROW0 = 4u << 27, - GEMM_TSS_MT8_ROW0 = 8u << 27, - GEMM_TSS_MT12_ROW0 = 12u << 27, -}; - -__attribute__((noinline, aligned(64))) void gemm_fp32( - float *C, const float *Bias, const float *Apack, const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi) -{ - const int ti = (int)ti_lo; - const int col_blocks = (int)gemm_l.N / 32; - const int col_tiles = (int)gemm_l.N / 16; // TILE_DIM - const int tile_stride = 16 * (int)gemm_l.K; - const int output_stride = gemm_l.N < 64 ? 64 : (int)gemm_l.N; - - // The M64 two-core partition assigns one row-tile pair to each core. - (void)ti_hi; - - if (ti_lo != 0) - asm volatile("nop" ::: "memory"); - - uintptr_t a0p = (uintptr_t)(Apack + ti * tile_stride); - uintptr_t a1p = (uintptr_t)(Apack + (ti + 1) * tile_stride); - uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); - uintptr_t tss0 = GEMM_TSS_MT0_ROW0; - uintptr_t tss4 = GEMM_TSS_MT4_ROW0; - uintptr_t tss8 = GEMM_TSS_MT8_ROW0; - uintptr_t tss12 = GEMM_TSS_MT12_ROW0; - // Bias remains tile-packed for two LMUL=8 loads per tile. VTSE writes C - // through independent row-major pointers. - uintptr_t q0 = (uintptr_t)(Bias + ti * col_tiles * 16 * 16); - uintptr_t q1 = (uintptr_t)(Bias + (ti + 1) * col_tiles * 16 * 16); - uintptr_t p00 = (uintptr_t)(C + ti * 16 * output_stride); - uintptr_t p01 = p00 + 16 * sizeof(float); - uintptr_t p10 = (uintptr_t)(C + (ti + 1) * 16 * output_stride); - uintptr_t p11 = p10 + 16 * sizeof(float); - const uintptr_t c_stride = (uintptr_t)(output_stride * sizeof(float)); - const uintptr_t tile_dim = 16; - const uintptr_t TK = 8; - uintptr_t block_counter = (uintptr_t)col_blocks; - uintptr_t loop_counter; - - asm volatile( - - // Every block enters the same bias-load and first-K-group schedule. - "10:\n" - // K-Group 0 - "msetmtypei 1, 2\n" - "msettn x0, %[tile_dim]\n" - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "msettm x0, %[tile_dim]\n" - "msettk x0, %[tk]\n" - // Load all 16 rows of mt0 before its first MAC. Each LMUL=8 load - // supplies eight physical vector registers, hence two loads per tile. - "vle32.v v24, (%[q0])\n" "addi %[q0], %[q0], 512\n" - "vtmv.t.v %[tss0], v24\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v25\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v26\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v27\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v28\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v29\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v30\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v31\n" "addi %[tss0], %[tss0], 1\n" - "vle32.v v8, (%[q0])\n" "addi %[q0], %[q0], -512\n" - "vtmv.t.v %[tss0], v8\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v9\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v10\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v11\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v12\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v13\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v14\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.t.v %[tss0], v15\n" "addi %[tss0], %[tss0], -15\n" - - // A0 is reused by mt0 and mt4. B0 remains at its current address - // because v16..v23 must be restored after loading the mt8 bias. - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" - - // mt0 compute || load and install mt4 bias. - "vtfmm.tvv mt0, v0, v16\n" - "addi %[q0], %[q0], 1024\n" - "vle32.v v24, (%[q0])\n" "addi %[q0], %[q0], 512\n" - "vtmv.t.v %[tss4], v24\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v25\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v1, v17\n" - "vtmv.t.v %[tss4], v26\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v27\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtmv.t.v %[tss4], v28\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v29\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtmv.t.v %[tss4], v30\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v31\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v8, (%[q0])\n" "addi %[q0], %[q0], -512\n" - "vtmv.t.v %[tss4], v8\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v9\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtmv.t.v %[tss4], v10\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v11\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtmv.t.v %[tss4], v12\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v13\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtmv.t.v %[tss4], v14\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.t.v %[tss4], v15\n" "addi %[tss4], %[tss4], -15\n" - - // Replace the consumed mt4-bias group with B1 exactly once. B1 is - // then reused by mt4 and mt12. - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - - // mt4 compute || seed mt12 in v8..v15 and v16..v23. - "addi %[q1], %[q1], 1024\n" - "vle32.v v8, (%[q1])\n" "addi %[q1], %[q1], 512\n" - "vtfmm.tvv mt4, v0, v24\n" - "vtmv.t.v %[tss12], v8\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v9\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtmv.t.v %[tss12], v10\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v11\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - "vle32.v v16, (%[q1])\n" "addi %[q1], %[q1], -512\n" - "vtmv.t.v %[tss12], v12\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v13\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtmv.t.v %[tss12], v14\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v15\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtmv.t.v %[tss12], v16\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v17\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtmv.t.v %[tss12], v18\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v19\n" "addi %[tss12], %[tss12], 1\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtmv.t.v %[tss12], v20\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v21\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt4, v7, v31\n" - "vtmv.t.v %[tss12], v22\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.t.v %[tss12], v23\n" "addi %[tss12], %[tss12], -15\n" - - // mt12 compute || seed mt8 in v16..v23 and v0..v7. - "addi %[q1], %[q1], -1024\n" - "vle32.v v16, (%[q1])\n" "addi %[q1], %[q1], 512\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtmv.t.v %[tss8], v16\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v17\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtmv.t.v %[tss8], v18\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v19\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v0, (%[q1])\n" "addi %[q1], %[q1], -512\n" - "vtmv.t.v %[tss8], v20\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v21\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtmv.t.v %[tss8], v22\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v23\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtmv.t.v %[tss8], v0\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v1\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtmv.t.v %[tss8], v2\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v3\n" "addi %[tss8], %[tss8], 1\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtmv.t.v %[tss8], v4\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v5\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtmv.t.v %[tss8], v6\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.t.v %[tss8], v7\n" "addi %[tss8], %[tss8], -15\n" - - // mt8 compute. A0 may be preloaded because mt8 reads A1 from v8..v15. - // Keep B0 intact until the final v15/v23 consumer has issued. - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - - // K-Group 1 to N-3 - // vle B1 - // vtfmm mt0, A0, B0 - "9:\n" - "li %[loop], 5\n" - "1:\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "vle32.v v24, (%[b1p])\n" - "vtfmm.tvv mt0, v2, v18\n" - "addi %[b1p], %[b1p], 512\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtfmm.tvv mt0, v7, v23\n" - - // vle A1 - // vtfmm mt4, A0, B1 - "vle32.v v8, (%[a1p])\n" - "vtfmm.tvv mt4, v0, v24\n" - "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtfmm.tvv mt4, v7, v31\n" - - // preload vle A0 - // vtfmm mt8, A1, B0 - "vle32.v v0, (%[a0p])\n" - "vtfmm.tvv mt8, v8, v16\n" - "addi %[a0p], %[a0p], 512\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - - // preload vle B0 - // vtfmm mt12, A1, B1 - "addi %[loop], %[loop], -1\n" - "vle32.v v16, (%[b0p])\n" - "vtfmm.tvv mt12, v8, v24\n" - "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group K-2 & K-Group K-1 - // vle B0' - // vle A0' - // vtfmm mt0, A0, B0 - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "vtfmm.tvv mt0, v2, v18\n" - "vle32.v v24, (%[b0p])\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v8, (%[a0p])\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtfmm.tvv mt0, v7, v23\n" - "addi %[a0p], %[a0p], 512\n" - "addi %[b0p], %[b0p], 512\n" - - // vle B1 - // vtfmm mt0, A0', B0' - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v16, (%[b1p])\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - "addi %[b1p], %[b1p], 512\n" - - // vle B1' - // vtfmm mt4, A0, B1 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vle32.v v24, (%[b1p])\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtfmm.tvv mt4, v7, v23\n" - "addi %[b1p], %[b1p], 512\n" - - // vle A1 - // vtfmm mt4, A0', B1' || stage mt0 rows 0..7 in the VRF - "vtmv.v.t v0, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v1, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - - "vtmv.v.t v2, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v3, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - - "vtmv.v.t v4, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v5, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - - "vtmv.v.t v6, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v7, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - - "vtfmm.tvv mt4, v12, v28\n" - - "vtfmm.tvv mt4, v13, v29\n" - - "vtfmm.tvv mt4, v14, v30\n" - - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt4, v15, v31\n" - - // vle A1' - // vtfmm mt12, A1, B1 || stage mt0 rows 8..15 in the VRF - "vtmv.v.t v8, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v9, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v0, v16\n" - - "vtmv.v.t v10, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v11, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - - "vtmv.v.t v12, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v13, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - - "vtmv.v.t v14, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v15, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - - "vtfmm.tvv mt12, v4, v20\n" - - "vtfmm.tvv mt12, v5, v21\n" - - "vtfmm.tvv mt12, v6, v22\n" - - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt12, v7, v23\n" - - // vle reload B0 (restore B0 pointer) - // vtfmm mt12, A1', B1' || stage mt4 rows 0..7 in the VRF - "vtmv.v.t v16, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v17, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - - "vtmv.v.t v18, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v19, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - - "vtmv.v.t v20, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v21, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - - "vtmv.v.t v22, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v23, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - - "vtfmm.tvv mt12, v12, v28\n" - - "vtfmm.tvv mt12, v13, v29\n" - "addi %[b0p], %[b0p], -1024\n" - - "vtfmm.tvv mt12, v14, v30\n" - - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt12, v15, v31\n" - - // vle reload B0' - // vtfmm mt8, A1, B0 || stage mt4 rows 8..15 in the VRF - "vtmv.v.t v24, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v25, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v0, v16\n" - - "vtmv.v.t v26, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v27, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - - "vtmv.v.t v28, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v29, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - - "vtmv.v.t v30, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v31, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - - "vtfmm.tvv mt8, v4, v20\n" - - "vtfmm.tvv mt8, v5, v21\n" - - "vtfmm.tvv mt8, v6, v22\n" - - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt8, v7, v23\n" - - // vtfmm mt8, A1', B0' || stage all mt12 rows in the VRF - "addi %[loop], %[tss12], 8\n" - - "vtmv.v.t v0, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v1, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - - "vtmv.v.t v2, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v3, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - - "vtmv.v.t v4, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v5, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - - "vtmv.v.t v6, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v7, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - - "vtmv.v.t v16, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v17, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - - "vtmv.v.t v18, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v19, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - - "vtmv.v.t v20, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v21, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - - "vtmv.v.t v22, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v23, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "mv %[tss12], %[loop]\n" - - // The VTMV instructions above preserve the existing MAC schedule, - // but all four result tiles are committed through row-major VTSE. - "addi %[tss0], %[tss0], -16\n" - "li %[loop], 16\n" - "3:\n" - "vtse32 %[tss0], (%[p00])\n" - "add %[p00], %[p00], %[c_stride]\n" - "addi %[tss0], %[tss0], 1\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 3b\n" - - "addi %[tss4], %[tss4], -16\n" - "li %[loop], 16\n" - "4:\n" - "vtse32 %[tss4], (%[p01])\n" - "add %[p01], %[p01], %[c_stride]\n" - "addi %[tss4], %[tss4], 1\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 4b\n" - - "addi %[tss12], %[tss12], -16\n" - "li %[loop], 16\n" - "5:\n" - "vtse32 %[tss12], (%[p11])\n" - "add %[p11], %[p11], %[c_stride]\n" - "addi %[tss12], %[tss12], 1\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 5b\n" - - "addi %[blocks], %[blocks], -1\n" - "j 7f\n" - - // Complete the fourth row-major tile store for this block. - "7:\n" - "li %[loop], 16\n" - "2:\n" - "vtse32 %[tss8], (%[p10])\n" - "add %[p10], %[p10], %[c_stride]\n" - "addi %[tss8], %[tss8], 1\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 2b\n" - "beqz %[blocks], 8f\n" - - // Retarget the scalar streams only after the old mt8 store has - // consumed its p10/tss8 state. - "addi %[tss0], %[tss0], -16\n" - "addi %[tss4], %[tss4], -16\n" - "addi %[tss8], %[tss8], -16\n" - "addi %[tss12], %[tss12], -16\n" - - // Each VTSE stream advanced by 16 rows. Retarget it to the same row - // pair and the next 32-column output block. - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[p00], %[p00], %[loop]\n" - "sub %[p01], %[p01], %[loop]\n" - "sub %[p10], %[p10], %[loop]\n" - "sub %[p11], %[p11], %[loop]\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - - // q0 currently addresses this block's mt4 bias and q1 its mt8 bias. - // Advance both packed-bias row streams to the next 2x2 block. - "addi %[q0], %[q0], 1024\n" - "srli %[loop], %[loop], 1\n" - "add %[q1], %[q1], %[loop]\n" - "j 10b\n" - "8:\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss0] "+r"(tss0), - [tss4] "+r"(tss4), - [tss8] "+r"(tss8), - [tss12] "+r"(tss12), - [p00] "+r"(p00), - [p01] "+r"(p01), - [p10] "+r"(p10), - [p11] "+r"(p11), - [q0] "+r"(q0), - [q1] "+r"(q1), - [blocks] "+r"(block_counter), - [loop] "=&r"(loop_counter) - : - [c_stride] "r"(c_stride), - [tile_dim] "r"(tile_dim), - [tk] "r"(TK) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/kernel/gemm.h b/sw/vme/apps/tew32/fp32-matmul-vle-vtse/kernel/gemm.h deleted file mode 100644 index 34f8e936a..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/kernel/gemm.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// gemm.h — interface for the fp32×fp32=fp32 VME GEMM kernel. - -#include - -// Read the hardware cycle counter. -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; -} - -// Reading fcsr stalls Snitch while any Spatz scoreboard entry is live. Use it -// to turn an asynchronous kernel return into an architectural completion point. -static inline void wait_spatz(void) -{ - uint32_t fcsr; - asm volatile("csrr %0, fcsr" : "=r"(fcsr) :: "memory"); -} - -// FP32 VME GEMM over TE=16 with architectural tk=1 and RES_K=8 software -// residency. [ti_lo, ti_hi) selects an even-aligned range of 16-row tiles. -void gemm_fp32(float *C, const float *Bias, const float *Apack, - const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi); diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/main.c b/sw/vme/apps/tew32/fp32-matmul-vle-vtse/main.c deleted file mode 100644 index db16a7720..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/main.c +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// main.c — fp32×fp32=fp32 VME GEMM functional check - -#include -#include -#include - -#include DATAHEADER -#include "kernel/gemm.c" - -// Shared L1 buffers: core 0 allocates and DMAs; the pointers live in shared -// TCDM (.data) so every core sees the same addresses. Declared at file scope -// (not on main's stack) precisely so the assignment by core 0 is visible to the -// other cores after the barrier -- the sp-fmatmul multi-core pattern. -static const float *Apack, *Bpack, *Cref; -static float *Bias, *C; -static uint32_t allocation_failed; -static volatile uint32_t core_cycles[16]; - -static inline float gemm_bias(uint32_t row, uint32_t col) -{ - const int32_t value = (int32_t)((row * 3u + col * 5u) & 15u) - 8; - return (float)value * 0.03125f; -} - -int main(void) -{ - const uint32_t cid = snrt_cluster_core_idx(); - const uint32_t ncores = snrt_cluster_core_num(); -#ifdef SPATZ_GEMM_SINGLE_CORE - const uint32_t active_cores = 1; -#else - const uint32_t active_cores = ncores; -#endif - - const uint32_t M = gemm_l.M; - const uint32_t N = gemm_l.N; - const uint32_t K = gemm_l.K; - const uint32_t TM = 16; - const uint32_t TN = 16; - const uint32_t output_ld = N; - - // Core 0 owns the DMA engine and publishes all shared L1 pointers. - const uint32_t c_bytes = M * output_ld * sizeof(float); - const uint32_t bias_bytes = M * N * sizeof(float); - const uint32_t c_ref_bytes = M * N * sizeof(float); - const uint32_t a_bytes = K * M * sizeof(float); - const uint32_t b_bytes = K * N * sizeof(float); - - if (cid == 0) { - C = (float *)snrt_l1alloc(c_bytes); - Bias = (float *)snrt_l1alloc(bias_bytes); - Cref = (float *)snrt_l1alloc(c_ref_bytes); - Bpack = (float *)snrt_l1alloc(b_bytes); - Apack = (float *)snrt_l1alloc(a_bytes); - allocation_failed = C == NULL || Bias == NULL || Cref == NULL || - Apack == NULL || Bpack == NULL; - if (!allocation_failed) { - snrt_dma_start_1d((void *)Cref, gemm_C_dram, c_ref_bytes); - snrt_dma_start_1d((void *)Apack, gemm_Apack_dram, a_bytes); - snrt_dma_start_1d((void *)Bpack, gemm_Bpack_dram, b_bytes); - snrt_dma_wait_all(); - memset(C, 0, c_bytes); - for (uint32_t row = 0; row < M; ++row) { - for (uint32_t col = 0; col < N; ++col) { - const uint32_t tile_row = row / TM; - const uint32_t tile_col = col / TN; - const uint32_t row_in_tile = row % TM; - const uint32_t col_in_tile = col % TN; - const uint32_t packed_idx = - (tile_row * (N / TN) + tile_col) * (TM * TN) + - row_in_tile * TN + col_in_tile; - Bias[packed_idx] = gemm_bias(row, col); - } - } - } - } - - // Partition complete 2x2 tile blocks. This keeps the range pair-aligned and - // leaves excess cores idle when M=32 has only one row-tile pair. - const uint32_t npairs = (M / TM) / 2; - const uint32_t pairs_per_core = - (npairs + active_cores - 1) / active_cores; - uint32_t pair_lo = cid < active_cores ? cid * pairs_per_core : npairs; - uint32_t pair_hi = cid < active_cores ? pair_lo + pairs_per_core : npairs; - if (pair_hi > npairs) pair_hi = npairs; - if (pair_lo > npairs) pair_lo = npairs; - const uint32_t ti_lo = 2 * pair_lo; - const uint32_t ti_hi = 2 * pair_hi; - - // Barrier: all cores wait until the buffers are DMA'd in and pointers set. - snrt_cluster_hw_barrier(); - - if (allocation_failed) { - if (cid == 0) printf("GEMM allocation failed\n"); - return 2; - } - - if (K % 8 != 0) - return 2; - - // Synchronize the launch, but keep both barriers outside the measured - // interval. Each core measures only its own GEMM slice. - snrt_cluster_hw_barrier(); - if (cid < active_cores) { - uint32_t t0 = get_cycle(); - gemm_fp32(C, Bias, Apack, Bpack, ti_lo, ti_hi); - uint32_t t1 = get_cycle(); - core_cycles[cid] = t1 - t0; - } else { - core_cycles[cid] = 0; - } - - // Publish all per-core durations before core 0 selects the parallel - // wall-clock time, defined as the slowest core's kernel duration. - snrt_cluster_hw_barrier(); - - // Only core 0 verifies and reports. - if (cid != 0) - return 0; - - uint32_t cycles = 0; - for (uint32_t i = 0; i < active_cores; ++i) { - if (core_cycles[i] > cycles) - cycles = core_cycles[i]; - } - - enum { CE = 8 }; - const uint64_t useful_fmas = (uint64_t)M * N * K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE * active_cores; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - // Element-wise error check. - // fdiv.s is not supported by the Spatz FPU; use fmul.s to avoid it. - // Instead of rel_err = abs_err / den > tol, check abs_err > tol * den. - int n_err = 0; - float max_abs = 0.0f; - const float tol = 1.0e-3f; - - uint32_t row_errs[M]; - memset(row_errs, 0, sizeof(row_errs)); - for (uint32_t idx = 0; idx < M * N; idx++) { - const uint32_t row = idx / N; - const uint32_t col = idx % N; - const float ref = Cref[idx] + gemm_bias(row, col); - float diff = C[row * output_ld + col] - ref; - float abs_err = diff < 0.0f ? -diff : diff; - float den = ref < 0.0f ? -ref : ref; - if (den < 1.0e-9f) den = 1.0e-9f; - if (abs_err > max_abs) max_abs = abs_err; - if (abs_err > tol && abs_err > tol * den) { - row_errs[idx / N]++; - n_err++; - } - } - for (uint32_t r = 0; r < M; r++) { - if (row_errs[r]) - printf("row %u: %u errors\n", (unsigned)r, (unsigned)row_errs[r]); - } - - printf("\n=== VME fp32 GEMM %ux%ux%u (TM=%u TN=%u, %u core%s) ===\n", - (unsigned)M, (unsigned)N, (unsigned)K, - (unsigned)TM, (unsigned)TN, - (unsigned)active_cores, active_cores == 1 ? "" : "s"); - printf("Cycles: %u\n", (unsigned)cycles); - printf("Utilization: %u.%02u%%\n", (unsigned)(utilization_bp / 100), - (unsigned)(utilization_bp % 100)); - printf("MaxAbsErr: 0x%08x\n", *(unsigned *)&max_abs); - printf("Errors: %d\n", n_err); - printf("Status: %s\n", n_err == 0 ? "PASS" : "FAIL"); - - // No trailing barrier: the non-zero cores have already returned, so an extra - // barrier here would have no partner and deadlock. - return n_err == 0 ? 0 : 1; -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vle-vtse/script/gen_data.py deleted file mode 100644 index 637655fc8..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/script/gen_data.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_gemm_layer gemm_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - f" .TM = {TM},", - f" .TN = {TN},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float gemm_C_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = gemm_Apack_dram + (mt*(K/4)+g)*64.", - f'static float gemm_Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = gemm_Bpack_dram + (nt*(K/4)+g)*64.", - f'static float gemm_Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vse/CMakeLists.txt index b69103094..29db4ab03 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/CMakeLists.txt +++ b/sw/vme/apps/tew32/fp32-matmul-vse/CMakeLists.txt @@ -42,16 +42,18 @@ function(add_fp32_vse_variant precision variant source_dir) list(GET _dims 1 _n) list(GET _dims 2 _k) - # main.c reserves two input panels when they fit in the 512-KiB TCDM. - # This is computed per shape so te32/K256 can fall back to one panel. + # Generate only variants whose two input panels fit in TCDM. math(EXPR _panel_bytes "32 * ${_tile_edge} * ${_k} + 64 * ${_tile_edge} * ${_tile_edge}") math(EXPR _double_buffer_bytes "2 * ${_panel_bytes}") - if(_double_buffer_bytes GREATER 524288) - set(_buffer_count 1) - else() - set(_buffer_count 2) + if(_double_buffer_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping FP32 VSE ${precision}/${variant} M${_m} N${_n} K${_k}: " + "double buffering requires ${_double_buffer_bytes} bytes, " + "but TCDM provides ${SNRT_TCDM_SIZE} bytes") + continue() endif() + set(_buffer_count 2) set(_variant_build_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${precision}/${variant}/K${_k}") @@ -83,21 +85,18 @@ function(add_fp32_vse_variant precision variant source_dir) add_custom_target(generate-${_target} DEPENDS "${_header}") add_snitch_test(${_target} "${source_dir}/main.c") add_dependencies(${_test_target} generate-${_target}) - set_target_properties(${_test_target} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/vme/apps") target_include_directories(${_test_target} PRIVATE "${source_dir}" "${source_dir}/kernel" "${source_dir}/data") target_compile_definitions(${_test_target} PRIVATE - MATMUL_K_CHUNK=${_k} MATMUL_BUFFER_COUNT=${_buffer_count} DATAHEADER="${_data_header}") endif() endforeach() endfunction() -foreach(_precision IN ITEMS te16 te32) +foreach(_precision IN ITEMS ${VME_TILE_VARIANT}) if(_precision STREQUAL "te16") # te16 is the active validation target. Keep all rolled scheduling # variants aligned with the passing VTSE target matrix. @@ -110,6 +109,9 @@ foreach(_precision IN ITEMS te16 te32) add_fp32_vse_variant(${_precision} rolled-vsetvl "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/vsetvl" ${_fp32_vse_te16_shapes}) + add_fp32_vse_variant(${_precision} rolled-store-group + "${CMAKE_CURRENT_SOURCE_DIR}/${_precision}/rolled/store-group" + ${_fp32_vse_te16_shapes}) # The te16 unrolled kernels are specialized by K. Keep their # existing source hierarchy and expose the same K64/K128/K256 ELF diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c index 6c1fce2ce..544fc122c 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -41,20 +38,20 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - const uintptr_t tile_bytes = operand_bytes * K; + const uintptr_t tile_bytes = operand_bytes * K_CHUNK; const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; // The first unrolled K-group is intentionally omitted from the middle // loop schedule used by this store-afterK variant. - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( - "mv %[rows], %[row_blocks]\n" + "mv %[rows], %[ROW_BLOCKS]\n" ".Lrow_%=:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" ".Lcol_%=:\n" // K-Group 1 "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=2, twiden=1 @@ -476,8 +473,8 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [cols] "=&r"(col_counter), [loop] "=&r"(loop_counter) : - [row_blocks] "r"((uintptr_t)row_blocks), - [col_blocks] "r"((uintptr_t)col_blocks), + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [tile_bytes] "r"(tile_bytes), diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h index 3f7dcd44e..583b87d3f 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/kernel/matmul.h @@ -25,6 +25,29 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && + 2 * BUFFER_BYTES > SNRT_TCDM_SIZE) + ? 1 : MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/main.c index 65ab836f5..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/main.c @@ -19,45 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && - 2 * BUFFER_BYTES > TCDM_BYTES) - ? 1 : MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -80,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -101,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -174,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-afterK/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/data/layer.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/data/layer.h new file mode 100644 index 000000000..149698826 --- /dev/null +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/data/layer.h @@ -0,0 +1,30 @@ +// Copyright 2024 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#include + +/** + * @brief VME matmul layer configuration. + * + * Describes a C = A * B problem with dimensions M x N x K. + */ +typedef struct { + uint32_t M; + uint32_t N; + uint32_t K; +} matmul_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/kernel/matmul.c new file mode 100644 index 000000000..f99338ffa --- /dev/null +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/kernel/matmul.c @@ -0,0 +1,741 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +// fp32 optimal kernel for TE=16, CE=8, and 1x4 tiles layout. + +#include "matmul.h" + +__attribute__((noinline, aligned(64))) void matmul_fp32_crossblock( + const float *At, const float *B, float *C) +{ + const uintptr_t c_stride = 64 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 1u; + const uintptr_t vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + const uintptr_t operand_bytes = TE * sizeof(float); + const uintptr_t operand_group_bytes = 4 * TE * sizeof(float); + + // Keep pointer updates inside each asm region to avoid scalar spills and tail merging. + // Pair rows 0/16, then rows 32/48. No generic block-coordinate division. + #pragma clang loop unroll(disable) + for (uint32_t nb = 0; nb < 4; nb += 2) { + const uint32_t mb0 = nb * TE; + uintptr_t old_c_ptr; + { + // First block: complete rows0-11 and retain rows12-15. + uintptr_t a0p = (uintptr_t)(At + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)B; + uintptr_t b1p = (uintptr_t)(B + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(B + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(B + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile( + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n addi %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n addi %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtzero mt0\n" + "vtzero mt4\n" + "vtzero mt8\n" + "vtzero mt12\n" + "vtfmm.tvv mt0, v0, v16\n" + "vle32.v v20, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vle32.v v21, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vle32.v v22, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vle32.v v23, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v0, v20\n" + "vle32.v v24, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v1, v21\n" + "vle32.v v25, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v2, v22\n" + "vle32.v v26, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtfmm.tvv mt4, v3, v23\n" + "vle32.v v27, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v0, v24\n" + "vle32.v v28, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v1, v25\n" + "vle32.v v29, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v2, v26\n" + "vle32.v v30, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtfmm.tvv mt8, v3, v27\n" + "vle32.v v31, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v0, v28\n" + "vle32.v v0, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v1, v29\n" + "vle32.v v1, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v2, v30\n" + "vle32.v v2, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vtfmm.tvv mt12, v3, v31\n" + "vle32.v v3, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + : [a0p] "+&r"(a0p), + [b0p] "+&r"(b0p), + [b1p] "+&r"(b1p), + [b2p] "+&r"(b2p), + [b3p] "+&r"(b3p) + : [group_vl] "r"(group_vl), + [operand_group_bytes] "i"(operand_group_bytes), + [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl), + [operand_bytes] "i"(operand_bytes) + : "memory", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); + + #pragma clang loop unroll(disable) + for (uint32_t remaining = K_CHUNK / 4 - 2; + remaining != 0; --remaining) { + asm volatile( + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v20, (%[b1p])\n addi %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtfmm.tvv mt4, v0, v20\n" + "vtfmm.tvv mt4, v1, v21\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v24, (%[b2p])\n addi %[b2p], %[b2p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtfmm.tvv mt4, v2, v22\n" + "vtfmm.tvv mt4, v3, v23\n" + "vtfmm.tvv mt8, v0, v24\n" + "vtfmm.tvv mt8, v1, v25\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v28, (%[b3p])\n addi %[b3p], %[b3p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n addi %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtfmm.tvv mt8, v2, v26\n" + "vtfmm.tvv mt8, v3, v27\n" + "vtfmm.tvv mt12, v0, v28\n" + "vtfmm.tvv mt12, v1, v29\n" + "vtfmm.tvv mt12, v2, v30\n" + "vtfmm.tvv mt12, v3, v31\n" + : [b1p] "+&r"(b1p), + [b2p] "+&r"(b2p), + [b3p] "+&r"(b3p), + [b0p] "+&r"(b0p) + : [group_vl] "r"(group_vl), + [operand_group_bytes] "i"(operand_group_bytes), + [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl) + : "memory", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "v16", "v17", "v18", "v19"); + // v0-v3 can be replaced only after all four tiles consume them. + asm volatile( + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n addi %[a0p], %[a0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + : [a0p] "+&r"(a0p) + : [group_vl] "r"(group_vl), + [operand_group_bytes] "i"(operand_group_bytes), + [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl) + : "memory", "v0", "v1", "v2", "v3"); + } + + // Final K group: move completed tiles while the next tile computes. + asm volatile( + "vtfmm.tvv mt0, v0, v16\n" + "vle32.v v20, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vle32.v v21, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vle32.v v22, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vle32.v v23, (%[b1p])\n" + "vtfmm.tvv mt4, v0, v20\n" + "vle32.v v24, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtmv.v.t v4, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt4, v1, v21\n" + "vle32.v v25, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtmv.v.t v8, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt4, v2, v22\n" + "vle32.v v26, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtmv.v.t v12, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt4, v3, v23\n" + "vle32.v v27, (%[b2p])\n" + "vtmv.v.t v16, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt8, v0, v24\n" + "vle32.v v28, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtmv.v.t v5, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt8, v1, v25\n" + "vle32.v v29, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtmv.v.t v9, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt8, v2, v26\n" + "vle32.v v30, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtmv.v.t v13, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt8, v3, v27\n" + "vle32.v v31, (%[b3p])\n" + "vtmv.v.t v17, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt12, v0, v28\n" + "vtmv.v.t v6, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtfmm.tvv mt12, v1, v29\n" + "vtmv.v.t v10, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtfmm.tvv mt12, v2, v30\n" + "vtmv.v.t v14, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtfmm.tvv mt12, v3, v31\n" + "vtmv.v.t v18, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v7, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v11, %[tss12]\n addi %[tss12], %[tss12], 1\n" + // Rows 0-1 are complete in v4-v7/v8-v11; store them before moving rows 2-3. + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v4, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v8, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v15, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v19, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v12, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v16, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + : [b1p] "+&r"(b1p), + [b2p] "+&r"(b2p), + [tss0] "+&r"(tss0), + [b3p] "+&r"(b3p), + [tss4] "+&r"(tss4), + [tss8] "+&r"(tss8), + [tss12] "+&r"(tss12), + [ptr] "+&r"(ptr) + : [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl), + [operand_bytes] "i"(operand_bytes), + [group_vl] "r"(group_vl), + [c_stride] "i"(c_stride) + : "memory", "v20", "v21", "v22", "v23", "v24", "v4", "v25", "v8", "v26", "v12", "v27", "v16", "v28", "v5", "v29", "v9", "v30", "v13", "v31", "v17", "v6", "v10", "v14", "v18", "v7", "v11", "v15", "v19"); + + // Rows 4-11: four contiguous 64-element rows per batch. + do { + asm volatile( + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v4, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v5, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v6, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v7, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v8, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v9, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v10, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v11, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v4, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v8, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v12, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v13, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v14, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v15, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v16, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v17, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v18, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v19, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v12, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v16, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + : [tss0] "+&r"(tss0), + [tss4] "+&r"(tss4), + [tss8] "+&r"(tss8), + [tss12] "+&r"(tss12), + [ptr] "+&r"(ptr) + : [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl), + [group_vl] "r"(group_vl), + [c_stride] "i"(c_stride) + : "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); + } while (tss0 < TE - 4); + + old_c_ptr = ptr; + } + { + // Second block: alternate paired operand loads / paired tile moves with m1 stores. + // Retain four old rows per tile until they are captured before vtzero. + const uint32_t mb1 = (nb + 1) * TE; + uintptr_t a0p = (uintptr_t)(At + mb1 * K_CHUNK); + uintptr_t b0p = (uintptr_t)B; + uintptr_t b1p = (uintptr_t)(B + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(B + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(B + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb1 * 64); + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + // Each buffer holds four rows of one old tile. A/B remain in v0-v3/v16-v31. + uintptr_t old_tss0 = 12, old_tss4 = 0x2000000cu; + uintptr_t old_tss8 = 0x4000000cu, old_tss12 = 0x6000000cu; + uintptr_t old_store = old_c_ptr; + asm volatile( + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n addi %[a0p], %[a0p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n addi %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v4, %[old_tss0]\n addi %[old_tss0], %[old_tss0], 1\n" + "vtmv.v.t v5, %[old_tss0]\n addi %[old_tss0], %[old_tss0], 1\n" + "vtmv.v.t v6, %[old_tss0]\n addi %[old_tss0], %[old_tss0], 1\n" + "vtmv.v.t v7, %[old_tss0]\n" + : [a0p] "+&r"(a0p), + [b0p] "+&r"(b0p), + [old_tss0] "+&r"(old_tss0) + : [group_vl] "r"(group_vl), + [operand_group_bytes] "i"(operand_group_bytes), + [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl) + : "memory", "v0", "v1", "v2", "v3", "v16", "v17", "v18", "v19", "v4", "v5", "v6", "v7"); + // Old mt0 is fully captured; reuse its accumulator for the new block. + asm volatile( + "vtzero mt0\n" + "vtfmm.tvv mt0, v0, v16\n" + "vle32.v v20, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v21, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vse32.v v4, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vtmv.v.t v8, %[old_tss4]\n addi %[old_tss4], %[old_tss4], 1\n" + "vtmv.v.t v9, %[old_tss4]\n addi %[old_tss4], %[old_tss4], 1\n" + "vse32.v v5, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vle32.v v22, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vle32.v v23, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vse32.v v6, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtmv.v.t v10, %[old_tss4]\n addi %[old_tss4], %[old_tss4], 1\n" + "vtmv.v.t v11, %[old_tss4]\n" + "vse32.v v7, (%[old_store])\n" + : [b1p] "+&r"(b1p), + [old_store] "+&r"(old_store), + [old_tss4] "+&r"(old_tss4) + : [operand_bytes] "i"(operand_bytes), + [c_stride] "i"(c_stride) + : "memory", "v20", "v8", "v21", "v9", "v22", "v10", "v23", "v11"); + // Old mt4 is fully captured; reuse its accumulator for the new block. + asm volatile( + "vtzero mt4\n" + : + : + : "memory"); + old_store = old_c_ptr + 1 * TE * sizeof(float); + asm volatile( + "vtfmm.tvv mt4, v0, v20\n" + "vle32.v v24, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vle32.v v25, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vse32.v v8, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt4, v1, v21\n" + "vtmv.v.t v4, %[old_tss8]\n addi %[old_tss8], %[old_tss8], 1\n" + "vtmv.v.t v5, %[old_tss8]\n addi %[old_tss8], %[old_tss8], 1\n" + "vse32.v v9, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt4, v2, v22\n" + "vle32.v v26, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vle32.v v27, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vse32.v v10, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt4, v3, v23\n" + "vtmv.v.t v6, %[old_tss8]\n addi %[old_tss8], %[old_tss8], 1\n" + "vtmv.v.t v7, %[old_tss8]\n" + "vse32.v v11, (%[old_store])\n" + : [b2p] "+&r"(b2p), + [old_store] "+&r"(old_store), + [old_tss8] "+&r"(old_tss8) + : [operand_bytes] "i"(operand_bytes), + [c_stride] "i"(c_stride) + : "memory", "v24", "v4", "v25", "v5", "v26", "v6", "v27", "v7"); + // Old mt8 is fully captured; reuse its accumulator for the new block. + asm volatile( + "vtzero mt8\n" + : + : + : "memory"); + old_store = old_c_ptr + 2 * TE * sizeof(float); + asm volatile( + "vtfmm.tvv mt8, v0, v24\n" + "vle32.v v28, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vle32.v v29, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vse32.v v4, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt8, v1, v25\n" + "vtmv.v.t v8, %[old_tss12]\n addi %[old_tss12], %[old_tss12], 1\n" + "vtmv.v.t v9, %[old_tss12]\n addi %[old_tss12], %[old_tss12], 1\n" + "vse32.v v5, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt8, v2, v26\n" + "vle32.v v30, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vle32.v v31, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vse32.v v6, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt8, v3, v27\n" + "vtmv.v.t v10, %[old_tss12]\n addi %[old_tss12], %[old_tss12], 1\n" + "vtmv.v.t v11, %[old_tss12]\n" + "vse32.v v7, (%[old_store])\n" + : [b3p] "+&r"(b3p), + [old_store] "+&r"(old_store), + [old_tss12] "+&r"(old_tss12) + : [operand_bytes] "i"(operand_bytes), + [c_stride] "i"(c_stride) + : "memory", "v28", "v8", "v29", "v9", "v30", "v10", "v31", "v11"); + // Old mt12 is fully captured; reuse its accumulator for the new block. + asm volatile( + "vtzero mt12\n" + : + : + : "memory"); + old_store = old_c_ptr + 3 * TE * sizeof(float); + asm volatile( + "vtfmm.tvv mt12, v0, v28\n" + "vle32.v v0, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v16, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vse32.v v8, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt12, v1, v29\n" + "vle32.v v1, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v17, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vse32.v v9, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt12, v2, v30\n" + "vle32.v v2, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v18, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vse32.v v10, (%[old_store])\n addi %[old_store], %[old_store], %[c_stride]\n" + "vtfmm.tvv mt12, v3, v31\n" + "vle32.v v3, (%[a0p])\n addi %[a0p], %[a0p], %[operand_bytes]\n" + "vle32.v v19, (%[b0p])\n addi %[b0p], %[b0p], %[operand_bytes]\n" + "vse32.v v11, (%[old_store])\n" + : [a0p] "+&r"(a0p), + [b0p] "+&r"(b0p), + [old_store] "+&r"(old_store) + : [operand_bytes] "i"(operand_bytes), + [c_stride] "i"(c_stride) + : "memory", "v0", "v16", "v1", "v17", "v2", "v18", "v3", "v19"); + + #pragma clang loop unroll(disable) + for (uint32_t remaining = K_CHUNK / 4 - 2; + remaining != 0; --remaining) { + asm volatile( + "vtfmm.tvv mt0, v0, v16\n" + "vtfmm.tvv mt0, v1, v17\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v20, (%[b1p])\n addi %[b1p], %[b1p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vtfmm.tvv mt0, v3, v19\n" + "vtfmm.tvv mt4, v0, v20\n" + "vtfmm.tvv mt4, v1, v21\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v24, (%[b2p])\n addi %[b2p], %[b2p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtfmm.tvv mt4, v2, v22\n" + "vtfmm.tvv mt4, v3, v23\n" + "vtfmm.tvv mt8, v0, v24\n" + "vtfmm.tvv mt8, v1, v25\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v28, (%[b3p])\n addi %[b3p], %[b3p], %[operand_group_bytes]\n" + "vle32.v v16, (%[b0p])\n addi %[b0p], %[b0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtfmm.tvv mt8, v2, v26\n" + "vtfmm.tvv mt8, v3, v27\n" + "vtfmm.tvv mt12, v0, v28\n" + "vtfmm.tvv mt12, v1, v29\n" + "vtfmm.tvv mt12, v2, v30\n" + "vtfmm.tvv mt12, v3, v31\n" + : [b1p] "+&r"(b1p), + [b2p] "+&r"(b2p), + [b3p] "+&r"(b3p), + [b0p] "+&r"(b0p) + : [group_vl] "r"(group_vl), + [operand_group_bytes] "i"(operand_group_bytes), + [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl) + : "memory", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "v16", "v17", "v18", "v19"); + // v0-v3 can be replaced only after all four tiles consume them. + asm volatile( + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vle32.v v0, (%[a0p])\n addi %[a0p], %[a0p], %[operand_group_bytes]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + : [a0p] "+&r"(a0p) + : [group_vl] "r"(group_vl), + [operand_group_bytes] "i"(operand_group_bytes), + [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl) + : "memory", "v0", "v1", "v2", "v3"); + } + + // Final K group: move completed tiles while the next tile computes. + asm volatile( + "vtfmm.tvv mt0, v0, v16\n" + "vle32.v v20, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v1, v17\n" + "vle32.v v21, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v2, v18\n" + "vle32.v v22, (%[b1p])\n addi %[b1p], %[b1p], %[operand_bytes]\n" + "vtfmm.tvv mt0, v3, v19\n" + "vle32.v v23, (%[b1p])\n" + "vtfmm.tvv mt4, v0, v20\n" + "vle32.v v24, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtmv.v.t v4, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt4, v1, v21\n" + "vle32.v v25, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtmv.v.t v8, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt4, v2, v22\n" + "vle32.v v26, (%[b2p])\n addi %[b2p], %[b2p], %[operand_bytes]\n" + "vtmv.v.t v12, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt4, v3, v23\n" + "vle32.v v27, (%[b2p])\n" + "vtmv.v.t v16, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtfmm.tvv mt8, v0, v24\n" + "vle32.v v28, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtmv.v.t v5, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt8, v1, v25\n" + "vle32.v v29, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtmv.v.t v9, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt8, v2, v26\n" + "vle32.v v30, (%[b3p])\n addi %[b3p], %[b3p], %[operand_bytes]\n" + "vtmv.v.t v13, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt8, v3, v27\n" + "vle32.v v31, (%[b3p])\n" + "vtmv.v.t v17, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtfmm.tvv mt12, v0, v28\n" + "vtmv.v.t v6, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtfmm.tvv mt12, v1, v29\n" + "vtmv.v.t v10, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtfmm.tvv mt12, v2, v30\n" + "vtmv.v.t v14, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtfmm.tvv mt12, v3, v31\n" + "vtmv.v.t v18, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v7, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v11, %[tss12]\n addi %[tss12], %[tss12], 1\n" + // Rows 0-1 are complete in v4-v7/v8-v11; store them before moving rows 2-3. + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v4, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v8, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v15, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v19, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v12, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v16, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + : [b1p] "+&r"(b1p), + [b2p] "+&r"(b2p), + [tss0] "+&r"(tss0), + [b3p] "+&r"(b3p), + [tss4] "+&r"(tss4), + [tss8] "+&r"(tss8), + [tss12] "+&r"(tss12), + [ptr] "+&r"(ptr) + : [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl), + [operand_bytes] "i"(operand_bytes), + [group_vl] "r"(group_vl), + [c_stride] "i"(c_stride) + : "memory", "v20", "v21", "v22", "v23", "v24", "v4", "v25", "v8", "v26", "v12", "v27", "v16", "v28", "v5", "v29", "v9", "v30", "v13", "v31", "v17", "v6", "v10", "v14", "v18", "v7", "v11", "v15", "v19"); + + // Rows 4-15: four contiguous 64-element rows per batch. + do { + asm volatile( + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v4, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v5, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v6, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v7, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v8, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v9, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v10, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v11, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v4, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v8, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[tile_vl]\n" + "vtmv.v.t v12, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v13, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v14, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v15, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vtmv.v.t v16, %[tss0]\n addi %[tss0], %[tss0], 1\n" + "vtmv.v.t v17, %[tss4]\n addi %[tss4], %[tss4], 1\n" + "vtmv.v.t v18, %[tss8]\n addi %[tss8], %[tss8], 1\n" + "vtmv.v.t v19, %[tss12]\n addi %[tss12], %[tss12], 1\n" + "vsetvli x0, %[group_vl], e32, m4, ta, ma\n" + "vse32.v v12, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + "vse32.v v16, (%[ptr])\n addi %[ptr], %[ptr], %[c_stride]\n" + : [tss0] "+&r"(tss0), + [tss4] "+&r"(tss4), + [tss8] "+&r"(tss8), + [tss12] "+&r"(tss12), + [ptr] "+&r"(ptr) + : [mtype] "r"(mtype), + [vtype] "r"(vtype), + [tile_vl] "r"(tile_vl), + [group_vl] "r"(group_vl), + [c_stride] "i"(c_stride) + : "memory", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19"); + } while (tss0 < TE); + } + } +} + +__attribute__((noinline, aligned(64))) void matmul_fp32( + const float *At, const float *B, float *C) +{ + const uintptr_t c_stride = 64 * sizeof(float); + const uintptr_t mtype = (TE << 10) | (2u << 5) | 1u; + const uintptr_t vtype = 0xD0; // e32, m1: one vector per tile row + const uintptr_t tile_vl = TE; + const uintptr_t group_vl = 4 * TE; + const uintptr_t operand_group_bytes = 4 * TE * sizeof(float); + + // Process each 16-row output block independently; no state crosses blocks. + for (uint32_t nb = 0; nb < 4; nb += 1) { + const uint32_t mb0 = nb * TE; + { + // Initialize and compute one complete 16-row output block. + uintptr_t a0p = (uintptr_t)(At + mb0 * K_CHUNK); + uintptr_t b0p = (uintptr_t)B; + uintptr_t b1p = (uintptr_t)(B + TE * K_CHUNK); + uintptr_t b2p = (uintptr_t)(B + 2 * TE * K_CHUNK); + uintptr_t b3p = (uintptr_t)(B + 3 * TE * K_CHUNK); + uintptr_t ptr = (uintptr_t)(C + mb0 * 64); + uintptr_t tss0 = 0x00000000u; + uintptr_t tss4 = 0x20000000u; + uintptr_t tss8 = 0x40000000u; + uintptr_t tss12 = 0x60000000u; + + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v0, (%0)" : : "r"(a0p) : "v0", "v1", "v2", "v3", "memory"); a0p += operand_group_bytes; + asm volatile("vle32.v v16, (%0)" : : "r"(b0p) : "v16", "v17", "v18", "v19", "memory"); b0p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtzero mt0" : : : "memory"); + asm volatile("vtzero mt4" : : : "memory"); + asm volatile("vtzero mt8" : : : "memory"); + asm volatile("vtzero mt12" : : : "memory"); + + for (uint32_t kg = 0; kg + 1 < K_CHUNK / 4; ++kg) { + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v20, (%0)" : : "r"(b1p) : "v20", "v21", "v22", "v23", "memory"); b1p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v20" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v1, v21" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v24, (%0)" : : "r"(b2p) : "v24", "v25", "v26", "v27", "memory"); b2p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt4, v2, v22" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v3, v23" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v0, v24" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v1, v25" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v28, (%0)" : : "r"(b3p) : "v28", "v29", "v30", "v31", "memory"); b3p += operand_group_bytes; + asm volatile("vle32.v v16, (%0)" : : "r"(b0p) : "v16", "v17", "v18", "v19", "memory"); b0p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt8, v2, v26" : : : "memory"); + asm volatile("vtfmm.tvv mt8, v3, v27" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v0, v28" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v1, v29" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v2, v30" : : : "memory"); + asm volatile("vtfmm.tvv mt12, v3, v31" : : : "memory"); + // v0-v3 can be replaced only after all four tiles consume them. + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v0, (%0)" : : "r"(a0p) : "v0", "v1", "v2", "v3", "memory"); a0p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + } + + // Final K group: move each completed tile row to the vector registers. + asm volatile("vtfmm.tvv mt0, v0, v16" : : : "memory"); + asm volatile("vtfmm.tvv mt0, v1, v17" : : : "memory"); + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vle32.v v20, (%0)" : : "r"(b1p) : "v20", "v21", "v22", "v23", "memory"); b1p += operand_group_bytes; + asm volatile("vle32.v v24, (%0)" : : "r"(b2p) : "v24", "v25", "v26", "v27", "memory"); b2p += operand_group_bytes; + asm volatile("vle32.v v28, (%0)" : : "r"(b3p) : "v28", "v29", "v30", "v31", "memory"); b3p += operand_group_bytes; + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtfmm.tvv mt0, v2, v18" : : : "memory"); + asm volatile("vtfmm.tvv mt0, v3, v19" : : : "memory"); + asm volatile("vtfmm.tvv mt4, v0, v20" : : : "memory"); + asm volatile("vtmv.v.t v4, %0" : : "r"(tss0) : "v4", "memory"); ++tss0; + asm volatile("vtfmm.tvv mt4, v1, v21" : : : "memory"); + asm volatile("vtmv.v.t v8, %0" : : "r"(tss0) : "v8", "memory"); ++tss0; + asm volatile("vtfmm.tvv mt4, v2, v22" : : : "memory"); + asm volatile("vtmv.v.t v12, %0" : : "r"(tss0) : "v12", "memory"); ++tss0; + asm volatile("vtfmm.tvv mt4, v3, v23" : : : "memory"); + asm volatile("vtmv.v.t v16, %0" : : "r"(tss0) : "v16", "memory"); ++tss0; + asm volatile("vtfmm.tvv mt8, v0, v24" : : : "memory"); + asm volatile("vtmv.v.t v5, %0" : : "r"(tss4) : "v5", "memory"); ++tss4; + asm volatile("vtfmm.tvv mt8, v1, v25" : : : "memory"); + asm volatile("vtmv.v.t v9, %0" : : "r"(tss4) : "v9", "memory"); ++tss4; + asm volatile("vtfmm.tvv mt8, v2, v26" : : : "memory"); + asm volatile("vtmv.v.t v13, %0" : : "r"(tss4) : "v13", "memory"); ++tss4; + asm volatile("vtfmm.tvv mt8, v3, v27" : : : "memory"); + asm volatile("vtmv.v.t v17, %0" : : "r"(tss4) : "v17", "memory"); ++tss4; + asm volatile("vtfmm.tvv mt12, v0, v28" : : : "memory"); + asm volatile("vtmv.v.t v6, %0" : : "r"(tss8) : "v6", "memory"); ++tss8; + asm volatile("vtfmm.tvv mt12, v1, v29" : : : "memory"); + asm volatile("vtmv.v.t v10, %0" : : "r"(tss8) : "v10", "memory"); ++tss8; + asm volatile("vtfmm.tvv mt12, v2, v30" : : : "memory"); + asm volatile("vtmv.v.t v14, %0" : : "r"(tss8) : "v14", "memory"); ++tss8; + asm volatile("vtfmm.tvv mt12, v3, v31" : : : "memory"); + asm volatile("vtmv.v.t v18, %0" : : "r"(tss8) : "v18", "memory"); ++tss8; + asm volatile("vtmv.v.t v7, %0" : : "r"(tss12) : "v7", "memory"); ++tss12; + asm volatile("vtmv.v.t v11, %0" : : "r"(tss12) : "v11", "memory"); ++tss12; + asm volatile("vtmv.v.t v15, %0" : : "r"(tss12) : "v15", "memory"); ++tss12; + asm volatile("vtmv.v.t v19, %0" : : "r"(tss12) : "v19", "memory"); ++tss12; + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vse32.v v4, (%0)" : : "r"(ptr) : "memory"); ptr += c_stride; + asm volatile("vse32.v v8, (%0)" : : "r"(ptr) : "memory"); ptr += c_stride; + asm volatile("vse32.v v12, (%0)" : : "r"(ptr) : "memory"); ptr += c_stride; + asm volatile("vse32.v v16, (%0)" : : "r"(ptr) : "memory"); ptr += c_stride; + + // Rows 4-15: four contiguous 64-element rows per batch. + // Keep the store epilogue compact for the instruction cache. + for (uint32_t row = 4; row < TE; row += 4) { + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v4, %0" : : "r"(tss0) : "v4", "memory"); ++tss0; + asm volatile("vtmv.v.t v5, %0" : : "r"(tss4) : "v5", "memory"); ++tss4; + asm volatile("vtmv.v.t v6, %0" : : "r"(tss8) : "v6", "memory"); ++tss8; + asm volatile("vtmv.v.t v7, %0" : : "r"(tss12) : "v7", "memory"); ++tss12; + asm volatile("vtmv.v.t v8, %0" : : "r"(tss0) : "v8", "memory"); ++tss0; + asm volatile("vtmv.v.t v9, %0" : : "r"(tss4) : "v9", "memory"); ++tss4; + asm volatile("vtmv.v.t v10, %0" : : "r"(tss8) : "v10", "memory"); ++tss8; + asm volatile("vtmv.v.t v11, %0" : : "r"(tss12) : "v11", "memory"); ++tss12; + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vse32.v v4, (%0)\n" "addi %0, %0, %1\n" + "vse32.v v8, (%0)\n" "addi %0, %0, %1\n" + : "+r"(ptr) : "i"(c_stride) : "memory"); + asm volatile("msetmtype %0, %1\n" "msettn x0, %2" : : "r"(mtype), "r"(vtype), "r"(tile_vl) : "memory"); + asm volatile("vtmv.v.t v12, %0" : : "r"(tss0) : "v12", "memory"); ++tss0; + asm volatile("vtmv.v.t v13, %0" : : "r"(tss4) : "v13", "memory"); ++tss4; + asm volatile("vtmv.v.t v14, %0" : : "r"(tss8) : "v14", "memory"); ++tss8; + asm volatile("vtmv.v.t v15, %0" : : "r"(tss12) : "v15", "memory"); ++tss12; + asm volatile("vtmv.v.t v16, %0" : : "r"(tss0) : "v16", "memory"); ++tss0; + asm volatile("vtmv.v.t v17, %0" : : "r"(tss4) : "v17", "memory"); ++tss4; + asm volatile("vtmv.v.t v18, %0" : : "r"(tss8) : "v18", "memory"); ++tss8; + asm volatile("vtmv.v.t v19, %0" : : "r"(tss12) : "v19", "memory"); ++tss12; + asm volatile("vsetvli x0, %0, e32, m4, ta, ma" : : "r"(group_vl) : "memory"); + asm volatile("vse32.v v12, (%0)\n" "addi %0, %0, %1\n" + "vse32.v v16, (%0)\n" "addi %0, %0, %1\n" + : "+r"(ptr) : "i"(c_stride) : "memory"); + } + } + } +} diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/kernel/matmul.h new file mode 100644 index 000000000..f659b2524 --- /dev/null +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/kernel/matmul.h @@ -0,0 +1,51 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: Pei-Yu Lin, ETH Zürich + +#pragma once + +#include + +enum { + CE = 8, + TE = 16, + CHUNK_SIZE = 4 * TE, + BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && + 2 * BUFFER_BYTES > SNRT_TCDM_SIZE) + ? 1 : MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); +void matmul_fp32_nocrossblock(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/main.c similarity index 52% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/main.c rename to sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/main.c index 2c29ce22b..d427b952f 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/main.c @@ -16,49 +16,28 @@ // // Author: Pei-Yu Lin, ETH Zürich -// main.c - fp32 x fp32 = fp32 VME GEMM functional check - #include -#include #include #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -enum { - TCDM_BYTES = 256 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = CHUNK_SIZE, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = 2, // 2 - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = L1_BUFFER_STRIDE + BUFFER_BYTES, -}; +static inline uint32_t get_cycle(void) +{ + uint32_t c; + asm volatile("csrr %0, mcycle" : "=r"(c)); + return c; +} _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, - "one M64xN64xK64 TEW32 buffer must fit in 128 KiB"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "two matmul buffers exceed the 256 KiB TCDM"); + "matmul panel exceeds its L1 buffer partition"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; -static uint8_t *l1_base; -static float *partials; -static float *c_out; - static inline uint32_t min_u32(uint32_t a, uint32_t b) { return a < b ? a : b; } @@ -77,16 +56,9 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static inline uint32_t cycle_count(void) { - uint32_t cycles; - asm volatile("csrr %0, mcycle" : "=r"(cycles)); - return cycles; -} -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, + uint32_t valid_tiles, uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && src_tile_k == K_CHUNK) { return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); @@ -99,57 +71,41 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, const size_t copy_bytes = valid_k * TE * sizeof(float); const size_t src_stride = src_tile_k * TE * sizeof(float); - // The assembly kernel advances between packed tiles by 16 * valid_k - // elements. Keep the L1 panel compact even though each buffer reserves - // space for the maximum K_CHUNK. return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); -} - -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - return (float)partial[(size_t)row * N_CHUNK + col]; + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * matmul_l.K + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * matmul_l.K + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -162,49 +118,49 @@ int main(void) { const uint32_t jobs = mb_count * nb_count; const uint32_t iterations = jobs * kb_count; - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); - c_out = (float *)snrt_l3alloc((size_t)matmul_l.M * matmul_l.N * - sizeof(float)); - if (l1_base == 0 || partials == 0 || c_out == 0) { - printf("fp16 matmul allocation failure\n"); + uint8_t *const l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); + float *const partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); + if (l1_base == 0 || partials == 0) { + printf("fp32 matmul allocation failure\n"); return 2; } - snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0, 0}; + snrt_dma_txid_t input_tid[L1_BUFFER_COUNT] = {0}; uint32_t current_buffer = 0; - input_tid[0] = fill_buffer(0, 0, 0, 0); + input_tid[0] = fill_buffer(l1_base, 0, 0, 0, 0); snrt_dma_wait(input_tid[0]); - if (iterations > 1) { + if (L1_BUFFER_COUNT == 2 && iterations > 1) { const uint32_t next_job = 1 / kb_count; - input_tid[1] = fill_buffer(1, next_job / nb_count, - next_job % nb_count, 1 % kb_count); + input_tid[L1_BUFFER_COUNT - 1] = fill_buffer( + l1_base, L1_BUFFER_COUNT - 1, next_job / nb_count, + next_job % nb_count, 1 % kb_count); } uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; - uint32_t fcsr; if (iter == 0) - start_cycle = cycle_count(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); - end_cycle = cycle_count(); + start_cycle = get_cycle(); + matmul_fp32((const float *)(base + A_OFFSET), + (const float *)(base + B_OFFSET), + (float *)(base + C_OFFSET)); + end_cycle = get_cycle(); - snrt_dma_start_1d(partials + (size_t)iter * C_ELEMENTS, - base + C_OFFSET, C_BYTES); + const snrt_dma_txid_t output_tid = snrt_dma_start_1d( + partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES); + if (L1_BUFFER_COUNT == 1) + snrt_dma_wait(output_tid); const uint32_t next_iter = iter + 1; - if (next_iter < iterations) { + if (next_iter < iterations && L1_BUFFER_COUNT == 1) { + const uint32_t next_job = next_iter / kb_count; + snrt_dma_wait(fill_buffer(l1_base, 0, next_job / nb_count, + next_job % nb_count, + next_iter % kb_count)); + } else if (next_iter < iterations) { const uint32_t next_buffer = current_buffer ^ 1; snrt_dma_wait(input_tid[next_buffer]); @@ -212,7 +168,7 @@ int main(void) { if (refill_iter < iterations) { const uint32_t refill_job = refill_iter / kb_count; input_tid[current_buffer] = fill_buffer( - current_buffer, refill_job / nb_count, + l1_base, current_buffer, refill_job / nb_count, refill_job % nb_count, refill_iter % kb_count); } current_buffer = next_buffer; @@ -220,7 +176,10 @@ int main(void) { } snrt_dma_wait_all(); + uint32_t errors = 0; + float max_abs = 0.0f; for (uint32_t row = 0; row < matmul_l.M; ++row) { + uint32_t row_errors = 0; for (uint32_t col = 0; col < matmul_l.N; ++col) { const uint32_t mb = row / M_CHUNK; const uint32_t nb = col / N_CHUNK; @@ -231,19 +190,10 @@ int main(void) { for (uint32_t kb = 0; kb < kb_count; ++kb) { const float *partial = partials + (size_t)(job * kb_count + kb) * C_ELEMENTS; - sum += partial_at(partial, local_row, local_col); + sum += partial[(size_t)local_row * N_CHUNK + local_col]; } - c_out[(size_t)row * matmul_l.N + col] = sum; - } - } - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t row = 0; row < matmul_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < matmul_l.N; ++col) { const size_t index = (size_t)row * matmul_l.N + col; - const float actual = c_out[index]; + const float actual = sum; const float expected = Cref_dram[index]; const float delta = actual > expected ? actual - expected : expected - actual; @@ -251,7 +201,7 @@ int main(void) { (expected < 0.0f ? -expected : expected); if (delta > max_abs) max_abs = delta; - if (delta > tolerance) { + if (!(delta <= tolerance)) { if (row_errors == 0) printf("row %u: errors at cols ", row); else diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/script/gen_data.py similarity index 79% rename from sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/script/gen_data.py rename to sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/script/gen_data.py index efa468aa3..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/script/gen_data.py @@ -7,13 +7,13 @@ # # # Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json +# python3 script/gen_data.py -c script/matmul_64_64_64.json # # Output: # data/data_{M}_{N}_{K}.h # # The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). +# Optional field: seed (default 42). import argparse import pathlib @@ -34,9 +34,8 @@ def main(): "-c", "--cfg", type=pathlib.Path, required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", + help="JSON/HJSON config file (e.g. script/matmul_64_64_64.json)", ) - parser.add_argument("-v", "--verbose", action="store_true") args = parser.parse_args() with args.cfg.open() as f: @@ -45,19 +44,13 @@ def main(): M = int(param["M"]) N = int(param["N"]) K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) seed = int(param.get("seed", 42)) - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - torch.manual_seed(seed) - np.random.seed(seed) A = torch.randn((M, K), dtype=torch.float32).numpy() B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() + C = (torch.from_numpy(A) @ torch.from_numpy(B)).numpy() # Contiguous k-fused-quad packing for the kcontig kernels. TK = 4 @@ -71,10 +64,6 @@ def main(): B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) .transpose(2, 0, 1, 3) # (nt, kg, slice, col) .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") lines = [ "// Copyright 2026 ETH Zurich and University of Bologna.", @@ -87,10 +76,16 @@ def main(): "", '#include "layer.h"', "", - "const vme_matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", + "const matmul_layer matmul_l = {", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/script/matmul_64_64_64.json similarity index 53% rename from sw/vme/apps/tew32/fp32-matmul-vle-vtse/script/gemm_64_64_64.json rename to sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/script/matmul_64_64_64.json index 3a26d4cde..dc71b114d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vle-vtse/script/gemm_64_64_64.json +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-group/script/matmul_64_64_64.json @@ -2,16 +2,13 @@ // Licensed under the Apache License, Version 2.0, see LICENSE for details. // SPDX-License-Identifier: Apache-2.0 // -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. +// Parameters for a MatMul { - kernel: "VME_GEMM", + kernel: "MatMul", M: 64, N: 64, K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) + TM: 16, + TN: 16, seed: 42 } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c index 618643673..5a4cfb415 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.c @@ -20,30 +20,27 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma const uintptr_t rewind_b0_bytes = operand_bytes * TE; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -486,14 +483,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" @@ -662,6 +659,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h index 3f7dcd44e..583b87d3f 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/kernel/matmul.h @@ -25,6 +25,29 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && + 2 * BUFFER_BYTES > SNRT_TCDM_SIZE) + ? 1 : MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/main.c index 65ab836f5..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/main.c @@ -19,45 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && - 2 * BUFFER_BYTES > TCDM_BYTES) - ? 1 : MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -80,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -101,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -174,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/store-interleave/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c index a479ef0bd..89bf0f9b1 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.c @@ -20,30 +20,27 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma const uintptr_t rewind_b0_bytes = operand_bytes * TE; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -414,14 +411,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" @@ -564,6 +561,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h index 3f7dcd44e..583b87d3f 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/kernel/matmul.h @@ -25,6 +25,29 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && + 2 * BUFFER_BYTES > SNRT_TCDM_SIZE) + ? 1 : MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/main.c index 65ab836f5..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/main.c @@ -19,45 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && - 2 * BUFFER_BYTES > TCDM_BYTES) - ? 1 : MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -80,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -101,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -174,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/rolled/vsetvl/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c index 853c7be94..5a82796d1 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K128/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c index fba9fb656..d6c3a9a35 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K256/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c index 286567546..30747a0c7 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_afterK/K64/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c index 7f90663e0..22b06a6c4 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K128/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c index 0acd231f8..a6fb71aa1 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K256/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c index 9b20f74e3..90e7946e1 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/store_interleave/K64/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c index aade22a5a..c4563b9f4 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -5085,6 +5084,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K128/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c index 95b153e89..992f4f304 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -9757,6 +9756,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K256/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c index 09cc79152..a06de1946 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -2749,6 +2748,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te16/unrolled/vsetvl/K64/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c index d1c9cd614..bcdcd8cb6 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -41,18 +38,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (1u << 5) | 1u; const uintptr_t matrix_vtype = 0xD1; // e32, m2, ta, ma - const uintptr_t tile_bytes = operand_bytes * K; + const uintptr_t tile_bytes = operand_bytes * K_CHUNK; const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 4 - 3; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( - "mv %[rows], %[row_blocks]\n" + "mv %[rows], %[ROW_BLOCKS]\n" ".Lrow_%=:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" ".Lcol_%=:\n" // K-Group 1 "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=2, twiden=1 @@ -322,8 +319,8 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [cols] "=&r"(col_counter), [loop] "=&r"(loop_counter) : - [row_blocks] "r"((uintptr_t)row_blocks), - [col_blocks] "r"((uintptr_t)col_blocks), + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [tile_bytes] "r"(tile_bytes), diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h index 94b72e13b..9a4231641 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/script/gen_data.py index 13fb92600..ed06ea6c0 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-afterK/script/gen_data.py @@ -72,10 +72,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c index 4533b51bd..ed750cd92 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.c @@ -20,36 +20,33 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t store_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (1u << 5) | 1u; const uintptr_t matrix_vtype = 0xD1; // e32, m2, ta, ma const uintptr_t rewind_b0_bytes = 8 * operand_bytes; - const uintptr_t tile_bytes = operand_bytes * K; + const uintptr_t tile_bytes = operand_bytes * K_CHUNK; const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 4 - 3; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -340,7 +337,7 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "mv %[loop], %[col_c_back]\n" "j 5f\n" "4:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "add %[a0p], %[a0p], %[tile_bytes]\n" "add %[a1p], %[a1p], %[tile_bytes]\n" "sub %[b0p], %[b0p], %[row_b_rewind]\n" @@ -449,6 +446,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h index 94b72e13b..9a4231641 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/script/gen_data.py index 13fb92600..ed06ea6c0 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/store-interleave/script/gen_data.py @@ -72,10 +72,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c index eb21d2573..6fab1695d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uintptr_t tile_stride = (uintptr_t)TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; const uintptr_t operand_bytes_m2 = operand_bytes; @@ -38,8 +35,8 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t store_counter; uintptr_t vl_counter; @@ -47,7 +44,7 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f const uintptr_t matrix_mtype = (TE << 10) | (1u << 5) | 1u; const uintptr_t matrix_vtype = 0xD1; // e32, m2, ta, ma const uintptr_t rewind_b0_bytes = 2 * operand_bytes_m8; - const uintptr_t middle_k_groups = (uintptr_t)K / 4 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -273,14 +270,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 5\n" "sub %[loop], %[loop], %[c_block_bytes]\n" "sub %[loop], x0, %[loop]\n" @@ -387,6 +384,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h index 94b72e13b..af0a76df9 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/kernel/matmul.h @@ -25,6 +25,29 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && + 2 * BUFFER_BYTES > SNRT_TCDM_SIZE) + ? 1 : MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/main.c index 65ab836f5..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/main.c @@ -19,45 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = (MATMUL_BUFFER_COUNT > 1 && - 2 * BUFFER_BYTES > TCDM_BYTES) - ? 1 : MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -80,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -101,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -174,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/script/gen_data.py index 13fb92600..ed06ea6c0 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vse/te32/rolled/vsetvl/script/gen_data.py @@ -72,10 +72,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/CMakeLists.txt deleted file mode 100644 index f72be821b..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(rolled-store-afterK "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/data/data_64_64_64.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/data/data_64_64_64.h deleted file mode 100644 index bcd195157..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/data/data_64_64_64.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Auto-generated by script/gen_data.py — do not edit by hand. - -#include "layer.h" - -const vme_matmul_layer matmul_l = { - .M = 64, - .N = 64, - .K = 64, -}; - -// C[M*N] row-major fp32 golden reference, C = A*B -static float Cref_dram[64*64] __attribute__((section(".data"))) = -{11.479634284973145f, -6.046323299407959f, -4.815614700317383f, -2.040816068649292f, -8.940674781799316f, 1.9331291913986206f, 4.764830112457275f, 14.248414993286133f, -3.9397127628326416f, -2.1485483646392822f, -0.8390139937400818f, -1.3708610534667969f, 13.092900276184082f, 3.2064008712768555f, 5.635993480682373f, 0.7455204725265503f, 7.919027805328369f, -11.349638938903809f, -4.450965404510498f, 15.0212984085083f, 0.6696339845657349f, -0.1861145943403244f, 4.869348049163818f, 3.4079155921936035f, -3.9942312240600586f, 8.851547241210938f, 8.788155555725098f, 6.571914127562195e-05f, 9.69917106628418f, -8.461928367614746f, -8.80200481414795f, 4.7893218994140625f, -0.18487417697906494f, -11.189379692077637f, -12.456897735595703f, -3.528578758239746f, 6.636075019836426f, 4.215647220611572f, -2.5566506385803223f, -2.2709872722625732f, 24.085134506225586f, -3.519670009613037f, -1.5794446468353271f, 13.206070899963379f, -9.0339937210083f, -0.49932846426963806f, 8.33430290222168f, -3.4920732975006104f, 24.13520622253418f, -2.866286516189575f, -5.177119255065918f, 8.296358108520508f, -2.486847162246704f, 7.566592693328857f, -18.43720817565918f, 17.894702911376953f, -17.132667541503906f, 4.636127471923828f, -15.027854919433594f, 14.163813591003418f, -16.567644119262695f, 4.0814080238342285f, -19.760257720947266f, 0.5436081290245056f, -1.0178847312927246f, -0.959050178527832f, -7.164222717285156f, 0.9985886216163635f, -8.26685619354248f, -2.2400271892547607f, 0.6488896608352661f, -3.7606098651885986f, 6.696377277374268f, 1.2368918657302856f, 0.3883592486381531f, -2.765044689178467f, 1.5367969274520874f, 5.624364852905273f, -1.1590087413787842f, 5.226413249969482f, 8.801167488098145f, 0.28941860795021057f, 7.781001091003418f, 1.1585612297058105f, -15.459006309509277f, -3.4766643047332764f, -8.453717231750488f, 10.173603057861328f, 3.6514530181884766f, 1.6193315982818604f, 5.127340793609619f, -0.9438349008560181f, -3.7507288455963135f, 13.667619705200195f, -10.900826454162598f, 11.15420913696289f, -2.949625253677368f, -2.746349811553955f, -16.438350677490234f, -17.064266204833984f, -4.720984935760498f, 4.893701553344727f, -2.473956823348999f, 4.502293586730957f, -5.264101982116699f, -8.454588890075684f, -5.186229705810547f, 5.584354877471924f, -2.95721435546875f, -3.410273790359497f, -6.588162899017334f, -0.8277148604393005f, -2.7242908477783203f, 4.025758266448975f, -1.4152922630310059f, 2.386408805847168f, 7.04514741897583f, -0.22987337410449982f, -0.45179229974746704f, 12.081316947937012f, -5.206918716430664f, 5.094143390655518f, -5.665627956390381f, 10.579627990722656f, -6.4196953773498535f, -8.07258415222168f, -19.385873794555664f, -8.815452575683594f, 1.0405519008636475f, -15.006485939025879f, -7.755399227142334f, 10.724888801574707f, 6.764276504516602f, 8.178961753845215f, 7.572741985321045f, 18.397531509399414f, -1.7755249738693237f, -1.9764132499694824f, 2.622986078262329f, -3.577617645263672f, -0.9461591839790344f, 14.711101531982422f, 8.183825492858887f, -7.469543933868408f, 14.31879711151123f, -6.517776012420654f, 2.1930079460144043f, 6.8929572105407715f, -10.022858619689941f, -1.7278424501419067f, 12.308111190795898f, 13.900723457336426f, 7.140420913696289f, 12.356964111328125f, 4.42258358001709f, -0.5004820227622986f, -1.5587430000305176f, 15.559115409851074f, 0.4262810945510864f, -2.8206470012664795f, 15.083824157714844f, -4.9072794914245605f, 2.5857160091400146f, 4.584408283233643f, -17.988496780395508f, -11.013274192810059f, 2.8712387084960938f, 4.990177631378174f, 0.6336867213249207f, 8.750150680541992f, 18.924428939819336f, 3.4662630558013916f, -14.959811210632324f, 3.8478240966796875f, 17.260263442993164f, -13.368204116821289f, 2.653244972229004f, -4.103200435638428f, -12.37991714477539f, -0.2966856062412262f, 5.892507076263428f, 6.881699562072754f, -3.230393648147583f, -10.482145309448242f, -13.298931121826172f, -6.5993547439575195f, 0.7129412293434143f, -2.0269880294799805f, 4.382270336151123f, -6.041990756988525f, -17.212663650512695f, -6.112694263458252f, 13.97390365600586f, -4.114153861999512f, -0.9799354672431946f, -4.399726390838623f, -5.032261848449707f, 0.5352955460548401f, -1.2309777736663818f, 10.451981544494629f, -5.048009872436523f, 3.677152633666992f, 11.969364166259766f, -0.417408287525177f, -3.176297426223755f, 5.826728343963623f, -5.306847095489502f, 10.880280494689941f, 12.627052307128906f, -5.021015644073486f, 0.382880836725235f, -12.032983779907227f, -2.4724607467651367f, -2.954784393310547f, 4.853247165679932f, -2.217081308364868f, -7.221776485443115f, 5.463408470153809f, -0.44735994935035706f, -10.677308082580566f, 1.9355524778366089f, -2.376770496368408f, 1.6620814800262451f, -12.09306526184082f, 9.559584617614746f, -1.3697234392166138f, -8.889398574829102f, 13.613365173339844f, -6.308327674865723f, -2.4649970531463623f, 2.388371467590332f, 4.159750938415527f, 0.5139855742454529f, 4.015327453613281f, -3.1259841918945312f, -1.0106412172317505f, 3.4432032108306885f, 4.1760029792785645f, -6.856860637664795f, -2.4840879440307617f, 0.686843752861023f, -1.0297985076904297f, 2.4013655185699463f, -3.5610408782958984f, 0.8839467167854309f, -2.944474458694458f, 12.640097618103027f, -3.0358707904815674f, -8.915057182312012f, 20.95890998840332f, 11.799171447753906f, 0.5034058690071106f, -1.9294973611831665f, -1.5430312156677246f, -1.1323392391204834f, 10.141955375671387f, -0.7313078045845032f, 3.43355655670166f, 12.49581241607666f, -0.004268331453204155f, 10.47712230682373f, 19.086078643798828f, -5.440404891967773f, -6.789121150970459f, 0.7849673628807068f, -0.7528467178344727f, -3.794801712036133f, -7.670317649841309f, 14.326471328735352f, -9.476102828979492f, 6.799417972564697f, -10.577319145202637f, -12.32802963256836f, 3.4096364974975586f, -24.734689712524414f, 3.53064227104187f, -1.0564911365509033f, 1.1934951543807983f, 2.728233814239502f, -2.273710012435913f, 1.6628460884094238f, 2.3249359130859375f, 3.8170626163482666f, 4.500376224517822f, -10.710893630981445f, 11.639716148376465f, -4.631977558135986f, 14.680305480957031f, 4.689661979675293f, -21.389917373657227f, 5.577933311462402f, 10.157344818115234f, 1.3816673755645752f, -9.77694034576416f, -1.59405517578125f, -5.5923357009887695f, 4.891682147979736f, -7.728332996368408f, -5.265920639038086f, 8.699220657348633f, -16.415210723876953f, 15.636664390563965f, -9.588492393493652f, -0.5692959427833557f, 2.7607104778289795f, 1.7100213766098022f, 5.899023532867432f, 6.637477874755859f, 1.3234360218048096f, 11.593015670776367f, 3.8447062969207764f, -12.39452075958252f, 17.31923484802246f, -4.310085296630859f, 1.3943078517913818f, 3.690248966217041f, 1.5341496467590332f, -0.16133688390254974f, -5.661871910095215f, -4.113314628601074f, 6.673490524291992f, -28.29741668701172f, -3.9005439281463623f, 5.377161979675293f, 2.418168067932129f, 14.741388320922852f, 12.219460487365723f, 2.1486709117889404f, -0.9644702672958374f, 2.6650185585021973f, -5.7412190437316895f, -11.613107681274414f, -4.728487014770508f, 4.203025817871094f, 4.285702705383301f, -6.733584880828857f, -1.751605749130249f, 1.8439090251922607f, 3.607456684112549f, -7.146326541900635f, 10.433805465698242f, 2.1755292415618896f, 7.192582607269287f, 2.093909502029419f, -12.907740592956543f, 10.749999046325684f, -0.37183743715286255f, -13.879500389099121f, -2.7690041065216064f, -0.5802304744720459f, -2.454869508743286f, 6.290034770965576f, -1.3150807619094849f, 1.3150523900985718f, -7.275416851043701f, -4.452425479888916f, 13.685830116271973f, -10.66769790649414f, 1.0413782596588135f, -11.05685043334961f, 18.703815460205078f, -1.3163670301437378f, 4.386673450469971f, 6.897695541381836f, -10.802721977233887f, -1.752117395401001f, 5.392110824584961f, -6.665993690490723f, 6.647964000701904f, 6.248779296875f, -8.761367797851562f, 3.014192581176758f, -12.200126647949219f, -4.44820499420166f, -6.547129154205322f, 5.838053226470947f, -3.972179889678955f, 4.548210620880127f, -0.38630470633506775f, 0.557646632194519f, -7.725645065307617f, -3.0316765308380127f, -10.36661148071289f, 9.558512687683105f, 1.3280283212661743f, -0.6749752759933472f, 2.5804808139801025f, 14.323273658752441f, 3.224461317062378f, -11.820540428161621f, -3.7925498485565186f, -8.334758758544922f, 0.4022352397441864f, 7.823636531829834f, -3.6518750190734863f, 2.0640387535095215f, -2.0144593715667725f, 1.261433482170105f, -12.39039421081543f, 1.648199200630188f, -8.816394805908203f, -6.666141986846924f, 4.299948692321777f, 11.934805870056152f, -9.07494831085205f, 8.816624641418457f, -7.657290935516357f, -1.1243833303451538f, 3.259331464767456f, 2.445486545562744f, 0.27485233545303345f, 9.906669616699219f, 10.861600875854492f, 1.530747413635254f, 3.6094765663146973f, 6.081133842468262f, -6.803634166717529f, 9.30950927734375f, -7.500870704650879f, -0.20945091545581818f, -4.56763219833374f, 9.062397956848145f, -3.900595188140869f, 0.46193817257881165f, -4.673060417175293f, -0.03734937310218811f, -7.745742321014404f, -1.5532246828079224f, -0.17770543694496155f, 7.410264492034912f, -2.503795862197876f, -4.5144267082214355f, 1.2131376266479492f, 1.3187705278396606f, -1.5181187391281128f, 1.0714176893234253f, 8.04094409942627f, 0.9295154213905334f, 4.887729167938232f, 4.386360168457031f, 2.0568528175354004f, -11.794127464294434f, -8.06806755065918f, 1.753397822380066f, -17.30348777770996f, -9.977322578430176f, 1.4055453538894653f, -3.8515894412994385f, -5.236893653869629f, 14.334654808044434f, -2.8135969638824463f, 12.197455406188965f, 5.414585113525391f, -3.6971359252929688f, 3.15084171295166f, -5.809675216674805f, 1.7624473571777344f, -1.7521796226501465f, -4.359520435333252f, -6.7524309158325195f, -6.430643558502197f, 4.465829372406006f, 5.363595008850098f, -0.05738918483257294f, -1.2110704183578491f, -12.731424331665039f, 7.597854137420654f, 5.500415325164795f, -6.625348091125488f, -4.435070991516113f, 16.017005920410156f, 6.0957231521606445f, 3.6745896339416504f, 18.56254005432129f, -6.415927886962891f, 7.9389967918396f, 2.5001425743103027f, 13.855697631835938f, 3.3746798038482666f, -13.604127883911133f, 0.829696774482727f, -0.6029493808746338f, 5.70452880859375f, 2.657468795776367f, -2.0609846115112305f, -5.867642402648926f, 9.778928756713867f, 4.533025741577148f, 5.282003402709961f, -1.0826342105865479f, -6.271111965179443f, 0.7187356352806091f, 8.167415618896484f, -17.741748809814453f, 5.170361518859863f, 1.9452539682388306f, 0.710178554058075f, 15.03416633605957f, 5.8055901527404785f, 9.560342788696289f, -7.244373798370361f, -7.103370189666748f, 6.701330184936523f, -13.6785306930542f, -7.435286998748779f, -1.4204037189483643f, 3.6168243885040283f, -1.2848598957061768f, 10.019153594970703f, -5.542983531951904f, 7.791118621826172f, 10.698689460754395f, 0.5794566869735718f, -5.135282039642334f, -0.725027859210968f, 6.094788551330566f, 3.9960646629333496f, 9.213926315307617f, -1.1690841913223267f, -11.310675621032715f, 4.797519207000732f, 2.9180209636688232f, 17.760066986083984f, -3.3779516220092773f, -5.8081278800964355f, 11.280855178833008f, -4.242335319519043f, -9.20323657989502f, 5.160378456115723f, 3.6303231716156006f, 8.061834335327148f, 0.7347631454467773f, 10.492819786071777f, -0.8277621865272522f, 15.720081329345703f, 6.086870193481445f, 2.3116986751556396f, -8.257061958312988f, -11.12254524230957f, -4.629530906677246f, -1.0439260005950928f, 2.503861665725708f, 4.953763484954834f, -7.421053886413574f, -4.31266975402832f, -0.7821304202079773f, -9.519991874694824f, -5.7066450119018555f, 2.1456873416900635f, 0.6137762665748596f, 9.2358980178833f, -4.393798351287842f, 0.7774427533149719f, -6.750569820404053f, 6.697755813598633f, 1.7473117113113403f, -5.084836959838867f, -0.3822762370109558f, 3.1108381748199463f, 1.4163764715194702f, -2.868486166000366f, -2.0308327674865723f, 12.085240364074707f, 0.32344046235084534f, -8.123509407043457f, -8.63557243347168f, 0.37889865040779114f, 5.865474700927734f, -2.006664752960205f, -4.2740278244018555f, -3.2797470092773438f, 0.7500452995300293f, 6.715641021728516f, 1.116511583328247f, -5.29896879196167f, 15.738234519958496f, -1.072593331336975f, 7.1444854736328125f, -0.17794132232666016f, 0.2486240416765213f, 10.500697135925293f, 5.418370246887207f, -2.850708484649658f, 5.028833866119385f, -1.3049635887145996f, -3.1064016819000244f, 7.260385036468506f, -0.2315407395362854f, 13.63786506652832f, -3.968128204345703f, 5.425351619720459f, 5.989855766296387f, 0.4256654977798462f, -11.24907112121582f, -13.982461929321289f, 3.0191447734832764f, -2.6573190689086914f, 0.18543148040771484f, 2.4465410709381104f, 7.224967002868652f, 9.639179229736328f, 11.72110366821289f, -14.941984176635742f, 2.158461809158325f, -9.967697143554688f, 7.731217384338379f, 0.2547948360443115f, 8.561541557312012f, -1.320813775062561f, -6.716103553771973f, -13.876091003417969f, -10.98348331451416f, 4.823574066162109f, 5.995814323425293f, 13.67184829711914f, -3.9956705570220947f, 2.217562675476074f, 8.181840896606445f, 11.416173934936523f, -7.419627666473389f, -2.8480899333953857f, -3.8081743717193604f, -9.326810836791992f, 2.7183923721313477f, 3.146589994430542f, -3.0829880237579346f, -5.814912796020508f, 8.27266788482666f, -2.7244033813476562f, 1.7461737394332886f, 12.851394653320312f, 6.015487194061279f, -11.382539749145508f, -10.727778434753418f, 1.8263652324676514f, 4.965327262878418f, -4.425846576690674f, 1.1692211627960205f, 0.7969360947608948f, 0.08354203402996063f, 6.083127975463867f, 5.9863152503967285f, -6.197132587432861f, -0.453395277261734f, -1.288467288017273f, -6.85847806930542f, 6.487435340881348f, -5.534632682800293f, 13.624343872070312f, 1.5271718502044678f, -1.4509201049804688f, -12.175165176391602f, -9.207465171813965f, 9.6865873336792f, -4.8576860427856445f, 5.351866245269775f, -8.007092475891113f, -1.1575658321380615f, 13.337957382202148f, 11.559327125549316f, -15.20022201538086f, 0.6360299587249756f, -16.20244789123535f, 7.8914031982421875f, 7.922462463378906f, 0.49626535177230835f, 0.6453051567077637f, 17.201900482177734f, 4.8184661865234375f, -9.721735954284668f, -9.466944694519043f, -1.581121563911438f, -3.897400379180908f, 13.486580848693848f, -2.706510305404663f, 2.2266478538513184f, -10.676257133483887f, -8.26251220703125f, -3.0972604751586914f, 6.262456893920898f, -7.870155334472656f, -13.106574058532715f, 1.9502202272415161f, 7.944813251495361f, 12.406501770019531f, -14.935750007629395f, 14.834214210510254f, -6.443024635314941f, -7.034519195556641f, 19.392358779907227f, -6.4235076904296875f, 5.952810287475586f, 4.202585697174072f, -6.166481018066406f, 6.54275369644165f, 4.269073009490967f, -4.20452356338501f, -11.693081855773926f, 0.5500400066375732f, -6.326947212219238f, -8.8986234664917f, -15.315683364868164f, 11.211211204528809f, -3.6820297241210938f, -4.241352081298828f, -0.7907126545906067f, 9.467594146728516f, 3.63688588142395f, -3.874994993209839f, -9.040021896362305f, 0.5441036224365234f, 14.560470581054688f, -4.352148532867432f, 2.122553586959839f, 7.5770955085754395f, 10.166125297546387f, -2.813676118850708f, -5.165591716766357f, -10.61868953704834f, 2.6967086791992188f, 1.273497462272644f, 16.388473510742188f, 12.926139831542969f, -18.8586483001709f, -2.5625576972961426f, -1.4167602062225342f, 3.1684563159942627f, -0.9735271334648132f, -12.724577903747559f, -9.471477508544922f, 1.9890105724334717f, -5.327760696411133f, 4.15165901184082f, -0.2690247595310211f, -5.091151237487793f, -8.251236915588379f, -13.286526679992676f, 11.11539363861084f, -2.1142094135284424f, -8.37338924407959f, -4.229764461517334f, 2.3841516971588135f, 9.676994323730469f, -16.521343231201172f, 21.009220123291016f, -14.100972175598145f, -18.230031967163086f, -2.3318724632263184f, 6.4106621742248535f, 5.1233930587768555f, 3.14448618888855f, -2.510714292526245f, 4.122291088104248f, -16.625221252441406f, -0.8044806122779846f, -7.060884952545166f, 15.047514915466309f, 1.5442057847976685f, -1.132941484451294f, 1.3845758438110352f, 5.088216781616211f, 5.57691764831543f, -5.411052703857422f, 15.526586532592773f, -8.33289623260498f, 2.9822065830230713f, -1.6032358407974243f, -16.77972984313965f, -8.319680213928223f, 1.4303581714630127f, 16.107824325561523f, -0.7335758805274963f, -10.483888626098633f, 6.441460132598877f, 4.506333827972412f, 2.4685215950012207f, 4.076876640319824f, -11.192659378051758f, -4.523487091064453f, -7.707409858703613f, 11.638382911682129f, -8.234586715698242f, 3.4906229972839355f, -9.156494140625f, -2.1987483501434326f, -14.089797019958496f, -0.26744771003723145f, 9.441929817199707f, 9.555951118469238f, 3.5714211463928223f, -13.786009788513184f, 2.148815870285034f, -6.62591028213501f, 7.7097487449646f, -1.0936174392700195f, 6.26248836517334f, 4.983580589294434f, -14.675965309143066f, -6.113205909729004f, 6.947558879852295f, -8.564501762390137f, 12.769579887390137f, -2.9595448970794678f, -2.199409008026123f, 4.292818069458008f, 0.014189623296260834f, -13.7341890335083f, 9.055596351623535f, 9.658831596374512f, 11.737518310546875f, -1.8660013675689697f, -7.643200397491455f, -7.508113384246826f, 8.38414192199707f, 5.401965618133545f, 13.07395076751709f, 3.3191146850585938f, 4.307673931121826f, 2.2782421112060547f, 1.9879189729690552f, -8.233000755310059f, -9.047646522521973f, 7.657324314117432f, 4.247537612915039f, 11.054131507873535f, -4.79965353012085f, -0.3028814196586609f, 3.658170461654663f, 6.45146369934082f, -2.8840818405151367f, 2.958085536956787f, 0.6611911654472351f, -15.455345153808594f, 14.44206428527832f, -1.5460273027420044f, 7.024033546447754f, -4.981692790985107f, -1.476488709449768f, -11.16106128692627f, 5.7634100914001465f, -4.065712928771973f, -7.169682502746582f, 6.687571048736572f, 9.344539642333984f, 0.3659030795097351f, 9.137345314025879f, -2.647665023803711f, -1.841318964958191f, 5.127373695373535f, -10.907052993774414f, 12.13635540008545f, 7.434513568878174f, -4.468003273010254f, 3.7723612785339355f, 6.410868167877197f, 4.488480091094971f, -4.279177665710449f, 7.486018180847168f, 2.041238784790039f, 0.8295110464096069f, 6.553818702697754f, 0.889700710773468f, 4.766284465789795f, -10.556770324707031f, -4.710302352905273f, -4.570557594299316f, 2.7720491886138916f, -0.9912987351417542f, -14.845259666442871f, 9.316060066223145f, 1.0089187622070312f, 8.025344848632812f, -1.9029171466827393f, 7.934247016906738f, -9.479381561279297f, 2.46075439453125f, 4.621790409088135f, -9.061838150024414f, 5.576177597045898f, 1.5385044813156128f, -0.02164754830300808f, 3.5387630462646484f, 8.792508125305176f, 11.135085105895996f, 0.09514058381319046f, 2.4890284538269043f, 6.469166278839111f, -12.828638076782227f, -11.913107872009277f, -12.835579872131348f, -0.7011198997497559f, 7.332045555114746f, 8.766676902770996f, -5.067351341247559f, -0.9870837926864624f, -12.046849250793457f, -8.263047218322754f, 7.80837869644165f, -2.4639174938201904f, 4.959366798400879f, 1.3135488033294678f, 0.6050440073013306f, 4.289178848266602f, -1.3765535354614258f, -9.210429191589355f, -2.441307544708252f, -13.704766273498535f, 1.7074304819107056f, 8.250596046447754f, -9.176819801330566f, -2.5442323684692383f, -10.301264762878418f, -9.052530288696289f, -10.033407211303711f, -6.968941688537598f, -2.9384915828704834f, 11.104891777038574f, 12.18455696105957f, -1.7053841352462769f, -10.236660957336426f, 13.148673057556152f, 5.770162105560303f, -14.608895301818848f, -3.367353677749634f, -1.997019648551941f, 0.6409828662872314f, 10.021282196044922f, 5.919330596923828f, -1.6916216611862183f, 8.11469841003418f, 15.174899101257324f, 1.4809973239898682f, 7.514713764190674f, -5.768565654754639f, 2.8586220741271973f, 0.785350501537323f, 6.117953300476074f, 12.343249320983887f, 11.014166831970215f, -3.0343480110168457f, -3.784940004348755f, -0.5094264149665833f, -3.813234567642212f, -1.6921602487564087f, 9.909290313720703f, 4.26685905456543f, -0.35572367906570435f, 13.063310623168945f, 0.9254683256149292f, -0.6383225321769714f, -4.169843673706055f, -0.2921450138092041f, -9.779097557067871f, 0.32207998633384705f, 0.46088171005249023f, -1.499398112297058f, -11.92739486694336f, 6.62190055847168f, 16.76767349243164f, 7.850483417510986f, -6.002956867218018f, -6.0743865966796875f, 8.590424537658691f, -8.350720405578613f, -7.048295497894287f, 8.3378267288208f, 10.439396858215332f, 4.051339149475098f, 7.447882652282715f, 2.115618944168091f, -10.832535743713379f, 3.544856071472168f, -10.704363822937012f, -1.747441053390503f, 3.1754229068756104f, -4.96966028213501f, -10.882390975952148f, 2.8613874912261963f, -14.762873649597168f, -3.866286516189575f, -9.634818077087402f, -4.488661289215088f, -16.146400451660156f, -8.561323165893555f, 10.843777656555176f, 7.5432963371276855f, 8.12264347076416f, -7.4483113288879395f, -10.192276000976562f, -0.18046212196350098f, 11.086979866027832f, 28.458358764648438f, -5.693641662597656f, 6.983304500579834f, 4.484742164611816f, -7.9314494132995605f, 5.449891567230225f, 3.5121002197265625f, -10.800859451293945f, 2.2634434700012207f, -10.603031158447266f, 6.369093894958496f, 21.34639549255371f, 5.682496070861816f, -6.500676155090332f, -3.933079719543457f, 9.044882774353027f, -1.2291513681411743f, 2.2405006885528564f, 7.146441459655762f, -5.252052307128906f, 13.324518203735352f, -2.54026198387146f, -4.696802139282227f, -1.9487093687057495f, 1.8994616270065308f, -12.246219635009766f, -6.6798505783081055f, -7.5826520919799805f, -4.574042320251465f, -1.7202016115188599f, 2.292769193649292f, 10.378962516784668f, 6.300224781036377f, 6.089559555053711f, 2.338552713394165f, 10.752300262451172f, -9.21757984161377f, 3.5770480632781982f, 1.6553243398666382f, -5.025013446807861f, -1.0492916107177734f, 2.3430588245391846f, -2.3533082008361816f, 13.465797424316406f, 8.130950927734375f, -17.104875564575195f, 3.76094913482666f, 3.0935189723968506f, 1.6557022333145142f, -1.1840074062347412f, 13.991741180419922f, 18.070926666259766f, -1.3191888332366943f, -5.403342247009277f, 3.2503833770751953f, 11.285150527954102f, 9.816794395446777f, -5.216005802154541f, -1.5604654550552368f, 6.930087089538574f, 2.469757556915283f, 0.5377483367919922f, -6.6069655418396f, -2.20501971244812f, -7.6572394371032715f, -12.394196510314941f, 4.9394073486328125f, -18.88083267211914f, 16.13848114013672f, 2.5295422077178955f, -6.287371635437012f, 3.2773492336273193f, 3.9482178688049316f, 1.5670056343078613f, -0.5477782487869263f, -9.703927993774414f, 7.892303466796875f, 16.467126846313477f, 1.2177623510360718f, -3.8086254596710205f, -0.10376394540071487f, 2.0969502925872803f, -13.377406120300293f, 11.43333625793457f, 0.33095118403434753f, -4.543916702270508f, -8.336254119873047f, 10.293583869934082f, -5.526627063751221f, 7.208284854888916f, 1.3287492990493774f, -0.00872433464974165f, -4.405787467956543f, 24.910831451416016f, 5.813371658325195f, -2.3747384548187256f, -1.58162522315979f, 7.138689041137695f, -0.8299044370651245f, 1.7968316078186035f, -0.16036289930343628f, 7.750216007232666f, -7.1681060791015625f, -1.6841415166854858f, 8.755945205688477f, -6.4495062828063965f, -3.929804801940918f, -11.690154075622559f, 5.407276630401611f, -19.410442352294922f, -7.46274471282959f, -6.528491497039795f, 11.476672172546387f, -10.896162986755371f, 9.078445434570312f, 4.966217994689941f, 2.7781200408935547f, 8.466453552246094f, 6.95072078704834f, -14.296183586120605f, 5.738059997558594f, -0.7061849236488342f, -3.4425599575042725f, -0.2813775837421417f, 4.374084949493408f, -2.8933258056640625f, -10.747404098510742f, -2.676332473754883f, 3.489074230194092f, 9.93961238861084f, 5.222956657409668f, -17.090818405151367f, 1.7347217798233032f, 13.138232231140137f, 8.740431785583496f, 6.423698425292969f, 1.796844244003296f, 2.321765661239624f, 6.327892303466797f, 0.3434549570083618f, 4.261251926422119f, 8.633129119873047f, -9.238373756408691f, 2.2492949962615967f, 2.5688412189483643f, -10.994841575622559f, -2.4493515491485596f, -3.7998733520507812f, -7.384821891784668f, -3.2261850833892822f, 17.528900146484375f, 3.3615753650665283f, 9.745475769042969f, -8.878006935119629f, 5.870270252227783f, -3.883683919906616f, -0.0326266773045063f, 7.409327030181885f, 10.064127922058105f, 17.231494903564453f, -4.299467086791992f, 0.6162081956863403f, -3.9329636096954346f, 3.457488775253296f, 1.6008586883544922f, 14.369483947753906f, -7.038842678070068f, 2.9360132217407227f, -11.51530933380127f, -6.849390029907227f, 13.802511215209961f, -3.17645263671875f, 8.121315002441406f, -4.379727840423584f, -0.7692107558250427f, -7.983859539031982f, -2.9673516750335693f, -16.916322708129883f, -16.98297882080078f, -10.993241310119629f, 6.7799153327941895f, -8.828645706176758f, -8.281076431274414f, 0.38319116830825806f, -6.442864418029785f, 5.842170715332031f, 2.7757022380828857f, -0.3426856994628906f, 11.318920135498047f, 9.262431144714355f, -17.823097229003906f, -11.620147705078125f, 0.7059615254402161f, 4.266175746917725f, 13.950201988220215f, -9.230637550354004f, -0.817978024482727f, 7.5885210037231445f, 15.164800643920898f, 7.9170756340026855f, -8.626565933227539f, -11.744421005249023f, -8.501404762268066f, -5.230059623718262f, -4.577203273773193f, -2.578476667404175f, 4.45864725112915f, 11.191590309143066f, 0.7849674224853516f, 2.8678131103515625f, 19.044151306152344f, -5.121172904968262f, 1.8655592203140259f, 5.731597900390625f, -4.264453411102295f, 0.31350159645080566f, 16.386018753051758f, -9.369375228881836f, -7.727303981781006f, -5.3810529708862305f, 4.404619216918945f, -5.699888229370117f, -5.798112869262695f, 0.13061971962451935f, 2.239311456680298f, 16.179100036621094f, -7.417810440063477f, 2.5456221103668213f, -3.236337900161743f, 1.1458196640014648f, -0.1307709664106369f, -0.8587997555732727f, -4.094841480255127f, -7.875479698181152f, -2.913313865661621f, 3.096707344055176f, 1.1844000816345215f, 11.496950149536133f, 4.843856334686279f, -4.238936424255371f, -1.1538751125335693f, -3.270214080810547f, 11.011395454406738f, 6.255121231079102f, 2.79670786857605f, -2.360741376876831f, 6.709340572357178f, -0.9215971827507019f, 4.753165245056152f, 11.404985427856445f, -11.04754638671875f, 5.086482048034668f, -1.1718279123306274f, 1.6178005933761597f, -1.7486406564712524f, -2.919844627380371f, 7.490941047668457f, -0.875328004360199f, -5.549253940582275f, -8.682398796081543f, 12.354720115661621f, -6.518106937408447f, 3.3926303386688232f, -7.628448486328125f, -0.5480865240097046f, 2.757507085800171f, -5.0333638191223145f, -12.717732429504395f, 9.747604370117188f, -14.137182235717773f, 13.830326080322266f, -8.072053909301758f, 9.366421699523926f, 4.771058082580566f, -3.8767879009246826f, 12.098913192749023f, -1.3983639478683472f, -2.516928195953369f, 17.085342407226562f, 0.700747013092041f, -4.593225479125977f, 3.2615785598754883f, 4.119733810424805f, 8.652438163757324f, 7.159962177276611f, -6.102161407470703f, -7.0815653800964355f, -4.8683037757873535f, -8.63952350616455f, -3.9259283542633057f, 9.596209526062012f, -9.913541793823242f, 3.975876569747925f, -3.891753911972046f, 5.650779724121094f, -6.815282344818115f, 2.731234550476074f, -0.2151940017938614f, -4.856814384460449f, 0.45669957995414734f, -7.445348262786865f, 9.566651344299316f, 1.5104864835739136f, 11.227133750915527f, -0.11537183076143265f, 4.145509719848633f, 4.6820244789123535f, 6.9148335456848145f, 4.9869384765625f, 6.348864555358887f, 0.12999588251113892f, -12.564311981201172f, -4.362220764160156f, -2.023157835006714f, -5.578927516937256f, 10.822314262390137f, -8.657614707946777f, 11.476557731628418f, -7.976850509643555f, 8.056036949157715f, 3.8919501304626465f, -3.0189290046691895f, 0.370013564825058f, 7.211360454559326f, 6.979368209838867f, -5.752582550048828f, -18.53067398071289f, -0.06240836903452873f, 6.615059852600098f, 14.322349548339844f, 7.855193614959717f, -2.3387138843536377f, -7.839303970336914f, -1.904661774635315f, 1.820509672164917f, 0.5016192197799683f, -14.7440767288208f, 13.704774856567383f, 11.45584774017334f, 10.099801063537598f, 0.022373056039214134f, -2.2562215328216553f, -3.3610339164733887f, -18.4713134765625f, -10.077775955200195f, 8.24351978302002f, -11.9453125f, 1.6951130628585815f, -6.876453399658203f, -9.18753433227539f, -3.261322498321533f, -5.389860153198242f, 1.553828239440918f, -3.9102296829223633f, 6.239824295043945f, 0.8332391381263733f, -0.4328247308731079f, -8.079927444458008f, 3.283322334289551f, 2.487851619720459f, 3.38568115234375f, 1.1550346612930298f, 5.795931339263916f, -3.827712059020996f, -17.76996612548828f, -1.3455630540847778f, -8.596356391906738f, -1.9443339109420776f, -3.347691535949707f, -3.8022572994232178f, -1.9706828594207764f, -8.254030227661133f, 4.786689758300781f, -9.022886276245117f, -10.170120239257812f, -7.9390668869018555f, -5.307765960693359f, -1.4348095655441284f, -4.2725114822387695f, 1.155957579612732f, 7.248950004577637f, 1.4605406522750854f, 2.4984130859375f, -4.926766395568848f, -1.360730767250061f, -3.5312368869781494f, -2.6760661602020264f, 13.249751091003418f, -2.3633241653442383f, 0.16462773084640503f, 2.9756968021392822f, -0.7048916220664978f, 3.0361123085021973f, 2.4167332649230957f, -18.724096298217773f, -2.668437957763672f, 4.492554664611816f, -1.228598952293396f, 4.189570426940918f, 5.219515323638916f, 1.4071485996246338f, -9.033429145812988f, 5.843366622924805f, 9.23009967803955f, -1.3791900873184204f, 2.5881869792938232f, 6.0501861572265625f, 10.04381275177002f, -6.157512187957764f, 7.24165678024292f, -4.1937384605407715f, -8.702505111694336f, 4.425203800201416f, -11.129894256591797f, -3.9875574111938477f, 10.64867877960205f, 5.056559085845947f, -10.747085571289062f, 1.264474630355835f, 7.634984016418457f, -16.58453369140625f, 8.581056594848633f, 6.325976848602295f, 2.503847360610962f, -1.0785939693450928f, 9.530458450317383f, -6.161625862121582f, -7.124795436859131f, 7.468658447265625f, -3.5646467208862305f, -2.393043041229248f, 10.85168743133545f, -1.2992841005325317f, 1.1869677305221558f, 12.415637969970703f, -0.384156733751297f, 1.6292704343795776f, -9.484331130981445f, -1.8839619159698486f, 7.0338287353515625f, -7.619279861450195f, 1.280126690864563f, -3.183976888656616f, -3.0420706272125244f, -3.4128353595733643f, -11.282638549804688f, -2.9777684211730957f, 0.18747295439243317f, -9.763144493103027f, 5.693328857421875f, 6.713829040527344f, 6.432366847991943f, 6.775195121765137f, 15.871235847473145f, -5.289862632751465f, 4.2354960441589355f, -2.9994354248046875f, -1.6106607913970947f, -8.912858009338379f, -9.295467376708984f, -8.503449440002441f, -4.607176303863525f, -1.5842245817184448f, 0.7097844481468201f, 10.443241119384766f, -0.1091175451874733f, -0.7141821384429932f, 0.6914375424385071f, -3.269829750061035f, -8.911444664001465f, -1.1965527534484863f, -1.024635910987854f, 3.5703232288360596f, -1.7879712581634521f, -3.1247658729553223f, -3.874403476715088f, -5.389334678649902f, 15.912910461425781f, -1.5096516609191895f, 10.253766059875488f, 5.733814716339111f, 0.046457380056381226f, -4.8697710037231445f, -6.196042537689209f, 26.0557861328125f, 6.26172399520874f, -6.333352088928223f, 3.3930180072784424f, -10.041016578674316f, -4.201577663421631f, -17.489336013793945f, -7.753904342651367f, -3.3036975860595703f, 9.259572982788086f, 5.622028350830078f, 7.418932914733887f, -4.166983127593994f, 3.4362385272979736f, 7.763801574707031f, 10.344937324523926f, 11.575509071350098f, -6.547769069671631f, 7.228475570678711f, 0.27201998233795166f, 11.231034278869629f, 15.090219497680664f, -3.0461761951446533f, 2.7282047271728516f, 0.22180375456809998f, 0.6711186170578003f, 7.804783821105957f, 1.4017285108566284f, 8.923149108886719f, -12.729116439819336f, 7.182803153991699f, 3.5145866870880127f, -2.1636273860931396f, -22.26779556274414f, -11.131207466125488f, -3.5905392169952393f, 25.306209564208984f, -4.522180557250977f, 8.663426399230957f, -2.190467357635498f, -9.864779472351074f, 0.2012891173362732f, -6.670775890350342f, -4.750314712524414f, 11.050165176391602f, -5.542118549346924f, 9.413408279418945f, 2.308258295059204f, 6.050876617431641f, 5.7319722175598145f, 3.2697877883911133f, -5.117337226867676f, 6.679595947265625f, 3.9800209999084473f, -13.86974048614502f, 4.446991920471191f, -8.73182201385498f, -4.240108966827393f, -3.3107078075408936f, 5.727823734283447f, -1.7871451377868652f, -1.1543947458267212f, 3.274351119995117f, 8.465099334716797f, 4.256385326385498f, 1.6312638521194458f, -0.4266124665737152f, -10.066319465637207f, -1.1442700624465942f, -13.979047775268555f, 9.38019847869873f, 10.103825569152832f, 1.44222891330719f, -12.20266056060791f, -4.161468505859375f, 5.828897476196289f, 3.763794422149658f, 3.17692494392395f, -7.616631507873535f, -4.522240161895752f, 1.3924592733383179f, 3.9834253787994385f, 7.975732326507568f, -5.482377529144287f, -10.70419979095459f, -1.8837058544158936f, -5.173501968383789f, -0.19477716088294983f, -3.4711830615997314f, 10.08415699005127f, -1.9378299713134766f, 5.39734411239624f, -4.766819953918457f, -7.339071750640869f, 4.702202796936035f, 6.930264949798584f, 14.5244779586792f, 1.373946189880371f, -7.17212438583374f, 4.260952949523926f, 13.407155990600586f, 1.6655051708221436f, 9.432223320007324f, 10.111971855163574f, -19.2701473236084f, 7.098327159881592f, -10.078179359436035f, 2.730682373046875f, -6.775233268737793f, 2.490149974822998f, 6.198461532592773f, 0.2401915341615677f, 13.640817642211914f, 6.729548454284668f, -5.323953151702881f, 7.341331958770752f, 0.6824727058410645f, -1.65504789352417f, -13.688631057739258f, 4.6387176513671875f, 3.3633487224578857f, -13.678863525390625f, 5.560359477996826f, -2.417947769165039f, 6.356764316558838f, 11.856796264648438f, -4.349381446838379f, -13.190492630004883f, 0.9404691457748413f, -0.07949790358543396f, 5.1438212394714355f, -5.610644817352295f, 9.887959480285645f, -10.704916000366211f, -6.3003129959106445f, 1.457470178604126f, -5.060678005218506f, 3.0186398029327393f, 0.4794176518917084f, 5.93607759475708f, 5.506703853607178f, -1.0873303413391113f, 2.6558244228363037f, 5.770946502685547f, -4.9762678146362305f, 12.409749031066895f, -7.4365234375f, 8.309518814086914f, 0.3051859140396118f, -6.369776248931885f, -6.938343048095703f, 14.529619216918945f, -8.19320011138916f, 4.215144157409668f, -8.835214614868164f, 2.212118148803711f, 7.021667003631592f, 1.2244818210601807f, -6.6155571937561035f, -5.983504295349121f, 5.949681758880615f, -9.470444679260254f, 1.7039216756820679f, 4.452876567840576f, 3.5223419666290283f, 1.386846661567688f, 12.392443656921387f, 3.4754412174224854f, -3.757160186767578f, 2.243192672729492f, 1.5498485565185547f, -5.969877243041992f, -9.53218936920166f, -0.9648003578186035f, -4.1782612800598145f, -8.63139533996582f, 2.888253688812256f, -4.9197845458984375f, -2.6188461780548096f, -0.34100860357284546f, 1.08323073387146f, 0.20715828239917755f, 0.17625738680362701f, 0.8104458451271057f, 6.235602378845215f, 1.1863518953323364f, 2.2748658657073975f, -2.9206295013427734f, -7.237782001495361f, 7.247722625732422f, -4.8302412033081055f, -16.587038040161133f, 3.3862411975860596f, 5.526927471160889f, -2.761680841445923f, -5.178590774536133f, 12.247193336486816f, -13.684733390808105f, -5.715342998504639f, -8.004195213317871f, 11.39901065826416f, -8.808146476745605f, -11.446958541870117f, -5.106409549713135f, -2.114779472351074f, 0.827870786190033f, 7.99212121963501f, 16.132234573364258f, -7.876288414001465f, 4.045842170715332f, 4.89345121383667f, 7.8902387619018555f, 2.6189887523651123f, 1.4187779426574707f, -4.95357608795166f, 1.9183154106140137f, -10.423624038696289f, 5.698985576629639f, -10.64306640625f, 8.366253852844238f, -6.523881435394287f, -12.701069831848145f, 18.55314064025879f, -17.26905059814453f, -2.0632481575012207f, 8.771729469299316f, 6.320028781890869f, 2.5256295204162598f, 14.239582061767578f, 3.1535520553588867f, 4.2085394859313965f, 0.1063476949930191f, 2.853101968765259f, 7.213879585266113f, 7.257656097412109f, -5.45920467376709f, -4.268126010894775f, -1.759271502494812f, -0.4200115203857422f, 3.3460514545440674f, 3.7233402729034424f, 1.9605106115341187f, 10.866938591003418f, -3.5412983894348145f, 0.06446467339992523f, -13.610902786254883f, -0.3217482566833496f, -13.58945083618164f, 7.48920202255249f, -4.73434591293335f, -4.7902374267578125f, 15.362975120544434f, -2.182408571243286f, 5.474664211273193f, -9.107425689697266f, 1.9057763814926147f, 5.691718101501465f, -5.837611675262451f, 0.9479477405548096f, 5.760697841644287f, 0.09852147847414017f, 4.3325629234313965f, -1.1939945220947266f, -9.364533424377441f, 5.033231735229492f, 4.058061122894287f, -13.785080909729004f, 1.9617301225662231f, -15.00662899017334f, 1.1908645629882812f, 1.5933691263198853f, -3.296618938446045f, 0.3798801302909851f, 3.9930577278137207f, 13.144612312316895f, -19.245630264282227f, 3.366441011428833f, -9.155226707458496f, -4.203514099121094f, -6.379355430603027f, 3.052100896835327f, 3.3825178146362305f, -5.03127384185791f, -20.389877319335938f, 15.295291900634766f, -17.634958267211914f, 7.198425769805908f, -26.755468368530273f, -5.668976783752441f, 9.652477264404297f, 1.5629017353057861f, -3.2165329456329346f, -2.528200626373291f, 8.87691593170166f, 1.8550244569778442f, 12.378725051879883f, 0.00554652651771903f, 5.4776787757873535f, -0.582221508026123f, -3.155843734741211f, 12.177905082702637f, -0.9750184416770935f, -0.16764011979103088f, -19.229124069213867f, 5.547354221343994f, 14.744590759277344f, 7.076948642730713f, 11.192045211791992f, 8.736725807189941f, -10.47435188293457f, -1.8428961038589478f, 8.822990417480469f, 10.079815864562988f, -9.724599838256836f, 4.329649925231934f, 22.17165756225586f, -12.209203720092773f, 6.37702751159668f, -15.575246810913086f, 4.703880786895752f, -1.969223976135254f, 0.45949697494506836f, 0.20915208756923676f, 11.847763061523438f, -3.744934558868408f, -13.487935066223145f, 3.874084234237671f, -4.578466415405273f, -7.086031436920166f, -6.233429908752441f, -6.785592555999756f, -3.3219637870788574f, 8.725717544555664f, 8.299102783203125f, 12.64601993560791f, -5.9381513595581055f, 0.5933879613876343f, 10.182705879211426f, 5.442933559417725f, 1.8527637720108032f, -4.759885787963867f, -4.977668285369873f, -7.128690719604492f, -2.2508225440979004f, -5.206679344177246f, 1.6984038352966309f, 3.4979476928710938f, -0.9469919204711914f, -5.347060680389404f, 2.9652979373931885f, -3.192228317260742f, 4.303493499755859f, 9.290297508239746f, 9.933444023132324f, -5.329427719116211f, 2.070354700088501f, 2.8292171955108643f, 13.59816837310791f, -2.8414688110351562f, 2.1585311889648438f, 5.236742973327637f, -2.824960947036743f, -18.918386459350586f, 0.8947755098342896f, -7.695527076721191f, 1.7885832786560059f, -18.260061264038086f, -15.04204273223877f, -1.8974013328552246f, -9.819128036499023f, 8.226827621459961f, -5.237205505371094f, 4.667030334472656f, -13.77105712890625f, 4.586790084838867f, -0.390918493270874f, -4.865528106689453f, -5.15276575088501f, 8.497437477111816f, 1.938669204711914f, -4.695268154144287f, 8.078354835510254f, 4.665389537811279f, -8.80441665649414f, 10.505725860595703f, -7.680267810821533f, -1.3688803911209106f, -3.3594307899475098f, 5.555975437164307f, 11.535223007202148f, -0.9209914803504944f, 2.747661590576172f, -14.198168754577637f, -2.046473503112793f, 9.80955982208252f, 0.09400451183319092f, 7.27423095703125f, 8.030008316040039f, -7.390738487243652f, 0.04207156226038933f, 6.497010231018066f, 2.9516983032226562f, -5.786115646362305f, 6.287818908691406f, 14.243786811828613f, -8.465189933776855f, -1.7876099348068237f, -5.9232072830200195f, 5.831533908843994f, -6.165143013000488f, 6.243556499481201f, -3.9629876613616943f, 10.030241966247559f, 13.016631126403809f, -6.402768611907959f, -3.428553819656372f, -2.3323075771331787f, 0.22513322532176971f, 9.516219139099121f, 8.901734352111816f, 1.7534918785095215f, 6.5073628425598145f, -11.850318908691406f, 22.981639862060547f, 0.9134898781776428f, -6.708924293518066f, 8.352168083190918f, 6.139276027679443f, -11.110394477844238f, -12.100716590881348f, 3.7225136756896973f, -6.486250400543213f, 4.79709529876709f, 3.2657382488250732f, -6.960175514221191f, 4.827823162078857f, 3.0818111896514893f, 4.233963966369629f, -1.300526738166809f, 8.41450023651123f, 27.55376625061035f, 6.323548793792725f, 1.2947889566421509f, -4.953489303588867f, 1.9106656312942505f, 1.7545992136001587f, 4.909358024597168f, -2.6168389320373535f, -13.989197731018066f, 2.96791672706604f, -15.628793716430664f, -4.646886348724365f, 2.3803651332855225f, -0.38021320104599f, 7.955515384674072f, 10.828896522521973f, -4.704763412475586f, -2.4767675399780273f, 4.345104694366455f, 9.674301147460938f, 14.051787376403809f, -1.9404326677322388f, -6.1034440994262695f, 4.96730375289917f, 8.356350898742676f, -12.492217063903809f, 5.052522659301758f, -11.288028717041016f, -18.85377311706543f, -2.1773197650909424f, 3.268491268157959f, -11.059211730957031f, 12.636406898498535f, 8.842144966125488f, 4.771931171417236f, 13.92164134979248f, -9.583444595336914f, 18.409446716308594f, -7.4875359535217285f, 24.48175621032715f, 5.571856498718262f, 3.763981819152832f, 2.6589486598968506f, -6.752713680267334f, -14.821791648864746f, -4.6671833992004395f, -7.933231830596924f, -5.372705459594727f, -16.37500762939453f, 8.258382797241211f, 4.597978115081787f, -4.5285491943359375f, -16.733186721801758f, -15.706796646118164f, -9.255148887634277f, 5.370093822479248f, -2.010138988494873f, 3.0411553382873535f, 5.09183406829834f, -0.09162905812263489f, 5.09684419631958f, 7.237029075622559f, -1.3704605102539062f, 15.3106107711792f, -9.188556671142578f, -2.035550117492676f, -11.181009292602539f, -3.858069658279419f, 0.42419710755348206f, 10.823795318603516f, 2.402636766433716f, -2.0442917346954346f, 2.90970778465271f, -10.479278564453125f, -5.484250068664551f, -5.404007434844971f, 5.897701263427734f, -10.198234558105469f, 3.919404983520508f, 11.954742431640625f, -3.368873119354248f, -2.416825532913208f, -5.485574245452881f, -1.0831294059753418f, 12.187384605407715f, 9.810623168945312f, 2.734238624572754f, -3.1333184242248535f, -5.630797386169434f, -3.6640121936798096f, -5.316211223602295f, -5.74959135055542f, -0.2944042682647705f, -23.43301010131836f, -3.242238998413086f, 3.7050588130950928f, 10.571874618530273f, 10.871262550354004f, 0.4957568347454071f, -13.565269470214844f, -0.34055426716804504f, -5.27890682220459f, -8.633820533752441f, -4.941458702087402f, -1.589221715927124f, 0.5624170899391174f, 5.252312183380127f, -6.480530261993408f, 1.477312445640564f, 3.0334291458129883f, 1.7544362545013428f, -1.811592698097229f, 2.166680097579956f, -2.746460199356079f, -14.217162132263184f, 0.9965202212333679f, -3.636343479156494f, -4.826642990112305f, -3.4294586181640625f, -13.18789291381836f, 1.1160309314727783f, 0.03090650774538517f, 4.610648155212402f, -6.316239356994629f, 5.343184947967529f, -3.5397722721099854f, 13.128379821777344f, 6.051750659942627f, 3.017526626586914f, -0.22869278490543365f, 1.0593830347061157f, -4.756152153015137f, -9.622599601745605f, -14.170649528503418f, 2.4119977951049805f, 3.543015241622925f, -6.777645111083984f, -2.0697014331817627f, -1.148790717124939f, -5.165157318115234f, -2.193115472793579f, 2.605295181274414f, -12.414362907409668f, -16.186542510986328f, -5.055665969848633f, 14.077547073364258f, -9.300932884216309f, 4.403715133666992f, -3.700418710708618f, 1.266947865486145f, 4.215131759643555f, 10.175348281860352f, -0.7335017919540405f, -6.666941165924072f, 1.7698924541473389f, 2.5551509857177734f, 14.33661937713623f, -6.243782043457031f, -7.089589595794678f, 5.264889240264893f, 2.0064051151275635f, -1.1940221786499023f, -0.8845052719116211f, 4.728024959564209f, 9.143659591674805f, -11.243470191955566f, -3.881427764892578f, 1.0975744724273682f, 5.020593166351318f, -2.509296178817749f, -0.7040889263153076f, -8.826937675476074f, 9.25462818145752f, 0.6523112058639526f, 8.413361549377441f, -9.956643104553223f, 4.320390701293945f, -7.327249050140381f, 0.7185436487197876f, 5.529834747314453f, 3.7425594329833984f, 11.977640151977539f, 0.3199857473373413f, -1.2970579862594604f, -0.6297903656959534f, -6.748528480529785f, -1.8058745861053467f, 7.02766227722168f, -14.052090644836426f, 3.328584909439087f, -11.310714721679688f, -8.255908966064453f, -1.6174887418746948f, 2.040614604949951f, -7.178774833679199f, -8.185322761535645f, -12.005358695983887f, -1.7305762767791748f, -15.079358100891113f, -8.002707481384277f, -13.309523582458496f, -4.7150774002075195f, -7.385843276977539f, 3.19026517868042f, -0.39262112975120544f, 10.768528938293457f, -12.257975578308105f, -20.101083755493164f, -10.958288192749023f, 7.338348388671875f, -5.042275905609131f, -4.23043966293335f, -4.301355361938477f, 6.800642967224121f, 0.3120475709438324f, 10.304023742675781f, -13.633195877075195f, -0.5820647478103638f, 5.642172813415527f, -7.806666851043701f, 4.420014381408691f, -11.09785270690918f, -0.6035222411155701f, -3.352290630340576f, -11.287544250488281f, -9.170281410217285f, 4.307720184326172f, 5.357173442840576f, 9.775383949279785f, -4.904134273529053f, 0.8738774657249451f, -1.9923650026321411f, -13.437588691711426f, 4.374331951141357f, -6.224425315856934f, -7.205249786376953f, -1.2282158136367798f, 16.407073974609375f, 14.745433807373047f, 10.053177833557129f, -5.026731014251709f, 13.399908065795898f, -4.700345039367676f, -2.810746908187866f, 4.868433952331543f, 14.18775463104248f, 0.07687458395957947f, -1.2327467203140259f, -1.3728224039077759f, 7.709836483001709f, 3.666210889816284f, 4.985335826873779f, -12.233044624328613f, -5.2307329177856445f, -5.411324977874756f, 5.240880012512207f, -2.543198347091675f, -21.135807037353516f, -11.415369987487793f, 5.937788486480713f, -0.01360476203262806f, -15.29384708404541f, -23.956432342529297f, 1.6906027793884277f, 3.3760499954223633f, 9.199305534362793f, 12.791433334350586f, 4.497912883758545f, 3.4790523052215576f, -2.8718466758728027f, 4.7231268882751465f, -12.663939476013184f, 8.505736351013184f, 0.8276928663253784f, -6.634997367858887f, 3.213438034057617f, -12.446330070495605f, -0.040080517530441284f, 8.142585754394531f, -5.221867084503174f, -7.002190113067627f, -0.7204623818397522f, -11.662397384643555f, -1.66923189163208f, 2.7600550651550293f, -7.987179279327393f, 1.4977517127990723f, -1.5806363821029663f, -3.7816100120544434f, 0.5635890960693359f, 1.5413968563079834f, -1.8732370138168335f, 0.9285376667976379f, -21.148822784423828f, 19.22353744506836f, -2.7432806491851807f, -14.096722602844238f, 8.600056648254395f, -8.63492488861084f, -0.5446416139602661f, -9.644774436950684f, 1.5598598718643188f, 9.11766242980957f, 11.372527122497559f, -12.296475410461426f, 7.532688617706299f, -8.911373138427734f, 6.434876441955566f, 3.617435932159424f, 4.387059688568115f, 12.811718940734863f, -9.318404197692871f, -0.4164440929889679f, -4.88645601272583f, -9.689740180969238f, -7.375895023345947f, 3.4132399559020996f, -5.05049991607666f, -4.028200626373291f, -8.181990623474121f, -7.417090892791748f, -6.058056831359863f, 4.370342254638672f, 4.167178630828857f, 10.502394676208496f, -8.585572242736816f, -7.519354820251465f, 11.942482948303223f, 13.272440910339355f, -2.757564067840576f, 1.3308485746383667f, -2.266592264175415f, -6.452311992645264f, -8.549099922180176f, -6.8195576667785645f, -6.657449722290039f, -11.0411376953125f, -21.244892120361328f, -9.757211685180664f, 3.444068670272827f, 5.706812858581543f, -1.7444701194763184f, 8.521875381469727f, -5.228854179382324f, -5.382219314575195f, -6.763889789581299f, -11.332088470458984f, -5.202573299407959f, 1.0299209356307983f, -11.423958778381348f, -1.0182582139968872f, 12.390052795410156f, -10.38305950164795f, -5.015763282775879f, 10.922207832336426f, -2.7742760181427f, 16.766313552856445f, -7.944525241851807f, -2.8002591133117676f, 10.874272346496582f, -9.857773780822754f, 2.1176915168762207f, -7.479413032531738f, -1.7775098085403442f, 8.738555908203125f, -1.012858271598816f, -0.524698793888092f, -2.0892815589904785f, 2.1344618797302246f, 2.1381497383117676f, -8.60706615447998f, -3.834965944290161f, 5.380415916442871f, -5.791232109069824f, 12.458983421325684f, -2.1414995193481445f, -6.775130271911621f, 6.628716468811035f, -6.176581382751465f, 1.7703351974487305f, 4.631689071655273f, 6.3223066329956055f, 2.441192388534546f, -5.126554012298584f, -1.0228726863861084f, 2.649749517440796f, 2.357867956161499f, -2.623882532119751f, 0.692119300365448f, 7.061679840087891f, -1.3346446752548218f, 9.300801277160645f, -2.3639140129089355f, -14.160957336425781f, 4.440097808837891f, 1.5377870798110962f, -2.905083179473877f, -14.448901176452637f, 3.674952983856201f, -3.8605237007141113f, 8.271306037902832f, 6.405794620513916f, -21.742176055908203f, -11.455822944641113f, 1.7873941659927368f, -11.738869667053223f, 8.434774398803711f, -20.15655517578125f, -1.367598056793213f, 7.51062536239624f, -7.572475910186768f, -0.1449132263660431f, 0.14903344213962555f, 1.539661169052124f, -8.20009994506836f, -7.516574382781982f, 6.837973117828369f, -5.781085968017578f, 1.294516921043396f, -9.226051330566406f, 1.2180476188659668f, -4.204068183898926f, 15.320548057556152f, -6.9747209548950195f, 10.799696922302246f, 9.24210262298584f, 9.05672550201416f, -7.968204498291016f, -4.376702308654785f, -2.1221911907196045f, 0.1324523389339447f, 4.775428771972656f, -0.22519434988498688f, -7.35526180267334f, 0.1219637468457222f, -11.087621688842773f, -7.007081985473633f, 5.306663990020752f, -16.567237854003906f, -9.150774002075195f, 19.48033332824707f, 9.253161430358887f, 8.751167297363281f, 6.719225883483887f, 5.08009672164917f, -6.268258571624756f, 0.8974515795707703f, -12.448920249938965f, 19.20334815979004f, -12.337990760803223f, -2.6060383319854736f, 17.232389450073242f, -5.7053303718566895f, -2.461069107055664f, -9.090248107910156f, -2.2077951431274414f, 5.545995235443115f, 2.147212028503418f, 8.779653549194336f, -1.8891314268112183f, 10.41473388671875f, 5.098225116729736f, 3.8328142166137695f, 4.6025872230529785f, -3.5405564308166504f, 4.929543972015381f, -15.190320014953613f, -4.475381851196289f, 4.857414722442627f, -0.2330942153930664f, -9.112611770629883f, -14.155166625976562f, -14.117636680603027f, 0.3170005977153778f, -6.148338794708252f, 2.6847426891326904f, 1.3375723361968994f, -10.719864845275879f, 19.82689666748047f, -11.773765563964844f, 1.4471533298492432f, 5.438664436340332f, 10.833401679992676f, 5.91309928894043f, -4.8504557609558105f, -0.8866681456565857f, 4.502740383148193f, 8.76336669921875f, -7.2705078125f, -2.602914810180664f, 0.4528222978115082f, 0.2632019817829132f, -3.9463186264038086f, -6.615810394287109f, -12.216948509216309f, 18.130096435546875f, -11.216374397277832f, -9.20461368560791f, 2.837886333465576f, -9.17361068725586f, -13.872835159301758f, 1.4739644527435303f, -4.641720294952393f, -6.107040882110596f, -0.7774397730827332f, 2.1887550354003906f, -3.053284168243408f, 6.492974758148193f, -8.238874435424805f, -17.8471622467041f, -5.672789096832275f, -2.3216474056243896f, -1.9701411724090576f, -7.030838966369629f, -2.107118844985962f, -0.4771590828895569f, -1.8420956134796143f, -14.038286209106445f, -4.026478290557861f, 3.295030355453491f, 4.460195064544678f, 6.017807960510254f, 7.253128528594971f, -7.15041971206665f, 0.515494167804718f, 4.030392646789551f, -3.3914072513580322f, 0.44524064660072327f, -0.19258594512939453f, -0.7080201506614685f, -2.50250244140625f, 2.334340810775757f, -11.007967948913574f, -12.750214576721191f, -7.300755977630615f, 14.395295143127441f, -3.3914880752563477f, 8.228107452392578f, 9.74307918548584f, 0.77601557970047f, 2.347871780395508f, 9.640531539916992f, -4.919435024261475f, -7.991837024688721f, 0.03702786564826965f, 0.6593180894851685f, -8.348962783813477f, 4.535181999206543f, 3.7413909435272217f, -4.0721588134765625f, 7.057863712310791f, -1.0119569301605225f, -5.5510711669921875f, -11.117044448852539f, 5.491680145263672f, 12.420897483825684f, 12.342089653015137f, -13.135498046875f, 0.531191349029541f, -3.1611971855163574f, 2.2477285861968994f, -4.986516952514648f, 7.761373996734619f, 2.0133655071258545f, 9.318339347839355f, 8.271308898925781f, 2.2258410453796387f, 7.38371467590332f, 1.7697837352752686f, 7.604837894439697f, -5.065467357635498f, 15.182759284973145f, 10.54971694946289f, 14.205101013183594f, -15.358622550964355f, -7.208074569702148f, -1.3423936367034912f, 4.829193115234375f, -6.763162612915039f, -6.093154430389404f, -3.8595669269561768f, 10.870450019836426f, 5.670471668243408f, -9.437480926513672f, 2.4107041358947754f, -6.857131004333496f, 2.4801251888275146f, -5.085203170776367f, 15.741228103637695f, -18.639524459838867f, 0.10690624266862869f, 10.266143798828125f, 0.7210645079612732f, 2.90657114982605f, 4.488924980163574f, 11.784412384033203f, 10.836816787719727f, 11.812047004699707f, 0.9760425090789795f, 6.398825645446777f, -11.69657039642334f, 9.617613792419434f, -2.4754092693328857f, 2.3830978870391846f, 4.122246742248535f, -6.237287521362305f, 6.2666802406311035f, -7.420814514160156f, -4.996058940887451f, 6.019990921020508f, 8.033538818359375f, -3.3236873149871826f, -10.9536714553833f, 6.0252366065979f, -10.10529613494873f, 10.607632637023926f, -1.0954471826553345f, -7.890513896942139f, -6.947789669036865f, 18.596343994140625f, -10.062642097473145f, 9.224631309509277f, 12.48892593383789f, 11.919086456298828f, 10.027180671691895f, 8.543699264526367f, -8.874430656433105f, -2.0335042476654053f, -14.272953033447266f, 3.9183735847473145f, -8.344069480895996f, -11.0534086227417f, 2.4750025272369385f, -0.029886547476053238f, 5.07313871383667f, -26.66839599609375f, -8.733036041259766f, -4.485158920288086f, 9.778528213500977f, 5.770241737365723f, 12.092632293701172f, 1.5609368085861206f, 11.50508975982666f, -0.6498426795005798f, -9.589741706848145f, -27.592344284057617f, -0.43285009264945984f, 11.695541381835938f, -6.151013374328613f, 5.020718097686768f, -12.249852180480957f, -2.1764540672302246f, 6.811885833740234f, 16.927019119262695f, -10.076769828796387f, -17.306066513061523f, 4.369997978210449f, 1.4093303680419922f, -2.871824026107788f, 2.7676126956939697f, -6.874851226806641f, -2.597337484359741f, -9.963288307189941f, 10.695664405822754f, 5.037891387939453f, -6.623497009277344f, 5.716047286987305f, 4.312154769897461f, 4.035192489624023f, 2.1116819381713867f, 3.357062578201294f, 1.416142225265503f, 3.6634714603424072f, -4.913036823272705f, -5.307648181915283f, 11.786884307861328f, -1.3370147943496704f, 10.858242988586426f, 8.861958503723145f, 6.27660608291626f, 12.768265724182129f, 12.327997207641602f, -8.675048828125f, -3.4217770099639893f, 7.692430019378662f, 1.7838584184646606f, 12.854020118713379f, -11.746981620788574f, -9.584903717041016f, 8.365598678588867f, 12.973992347717285f, 5.4280571937561035f, -9.301798820495605f, 11.192365646362305f, -2.602813243865967f, 6.336965084075928f, 0.5768340229988098f, -10.169051170349121f, -8.006020545959473f, 13.611660957336426f, -5.536235332489014f, 3.161245346069336f, -9.059117317199707f, -2.214181423187256f, -5.4537811279296875f, 6.635800838470459f, -12.098894119262695f, 11.331655502319336f, 13.68090534210205f, 3.1792664527893066f, 1.3622552156448364f, 2.2142395973205566f, 0.2345779687166214f, 0.5814613103866577f, 4.289124011993408f, 4.538444995880127f, -7.362942695617676f, 2.874087333679199f, 3.61981201171875f, -5.420032501220703f, 8.720592498779297f, 4.290198802947998f, 12.945777893066406f, 5.480059623718262f, -7.588292598724365f, 5.661287784576416f, -3.8283472061157227f, -8.462838172912598f, 7.285812854766846f, -10.747210502624512f, -3.003866672515869f, 7.091235637664795f, -1.580762505531311f, 8.499457359313965f, 0.9311726689338684f, 2.294097661972046f, 7.290738582611084f, -4.4237260818481445f, 13.710098266601562f, -7.216895580291748f, 2.5698392391204834f, -4.586067199707031f, -5.298479080200195f, -3.6828322410583496f, -1.1227331161499023f, 6.324790954589844f, 0.32012125849723816f, -0.4937366843223572f, -1.9536423683166504f, -9.237577438354492f, 1.0877151489257812f, 12.129581451416016f, 4.084137916564941f, -2.9383089542388916f, -5.019631862640381f, 0.6925498247146606f, 6.584831714630127f, 0.4018475115299225f, 0.21690116822719574f, 1.1919173002243042f, 2.2553939819335938f, -1.0278080701828003f, -13.67727279663086f, -1.5738617181777954f, 2.352642297744751f, 3.2829935550689697f, 0.22515998780727386f, -5.764885902404785f, -3.821402072906494f, 7.599716663360596f, 4.4758172035217285f, -18.555747985839844f, 10.015554428100586f, -5.277355194091797f, 4.857947826385498f, -4.972664833068848f, -7.69036865234375f, 0.8392868041992188f, -26.074081420898438f, -8.01816463470459f, 17.92840576171875f, 4.751870632171631f, 1.765329360961914f, -4.687397003173828f, 8.163311958312988f, 4.182957172393799f, 8.034849166870117f, 1.3160547018051147f, -0.5854548215866089f, -8.705289840698242f, -0.8203011155128479f, -3.8902640342712402f, -3.927004814147949f, 10.755240440368652f, -2.724745035171509f, -5.045247554779053f, -13.067852973937988f, 15.737813949584961f, 1.6259382963180542f, 0.018999869003891945f, 1.839993953704834f, 2.3978917598724365f, 1.3756414651870728f, 10.791954040527344f, -4.205836772918701f, 7.3124308586120605f, 3.0303399562835693f, 18.43343734741211f, 2.922717571258545f, 7.7857747077941895f, 9.202900886535645f, -11.383536338806152f, 4.063195705413818f, 4.0933308601379395f, 9.472505569458008f, 0.6550918817520142f, 1.219048261642456f, -3.383958101272583f, -13.222259521484375f, 6.909287452697754f, -11.573260307312012f, 4.1832146644592285f, 6.807440280914307f, -12.13596248626709f, -16.246288299560547f, -11.497807502746582f, 8.08572769165039f, -12.573715209960938f, -4.854974746704102f, -2.2262701988220215f, 0.8039400577545166f, 11.886690139770508f, -15.268168449401855f, -8.579140663146973f, -2.6850054264068604f, 5.244555950164795f, -6.70413064956665f, -3.2002172470092773f, 6.914629936218262f, 0.3705389201641083f, 14.895472526550293f, -6.253382205963135f, -6.998981475830078f, 4.563458442687988f, 5.477730751037598f, -9.057239532470703f, 9.500064849853516f, 4.373735427856445f, -0.7703867554664612f, 13.951043128967285f, -5.243332862854004f, 2.8435473442077637f, -6.590154647827148f, 4.319807529449463f, -2.5442638397216797f, -13.913573265075684f, -10.475417137145996f, -2.8347291946411133f, 8.493965148925781f, -10.500249862670898f, -5.360538005828857f, -4.234364986419678f, 0.10634779930114746f, -1.185592532157898f, -9.338807106018066f, 2.1675655841827393f, 3.200042486190796f, 4.2734856605529785f, 5.105844020843506f, -7.451834201812744f, -11.285500526428223f, -2.5912656784057617f, -4.90095853805542f, -7.117650508880615f, 5.817300796508789f, -2.694540023803711f, -6.723092555999756f, 3.3572030067443848f, 9.950608253479004f, -2.062587261199951f, -2.237545967102051f, 5.24613094329834f, 18.30078887939453f, -7.8479437828063965f, -2.280181407928467f, 8.82515811920166f, 0.1949726939201355f, 4.303898811340332f, -3.4921815395355225f, 16.898700714111328f, 11.57261848449707f, -2.4016921520233154f, -3.2015914916992188f, 10.793314933776855f, -6.424396514892578f, 1.5789941549301147f, 6.381648540496826f, -21.38960838317871f, 0.6480197906494141f, -13.997282028198242f, 0.6987282037734985f, -7.389518737792969f, 10.640852928161621f, -1.155595064163208f, -2.187462091445923f, -2.0233631134033203f, -2.092860698699951f, 8.194184303283691f, 12.357358932495117f, 12.201729774475098f, 5.951974868774414f, 1.862755537033081f, -4.316686630249023f, 6.706587791442871f, 5.658695220947266f, -3.7403807640075684f, 4.848812580108643f, -1.1044867038726807f, 4.073699474334717f, -6.804827690124512f, -2.227973461151123f, -14.886821746826172f, 4.097927093505859f, -5.149636745452881f, 14.20467472076416f, 10.393085479736328f, 16.598787307739258f, 20.412975311279297f, 0.4636867642402649f, -12.83670711517334f, 5.011075496673584f, 3.2573134899139404f, 2.541311502456665f, -3.704630136489868f, -7.621809482574463f, 2.2266361713409424f, -6.034920692443848f, -4.907690525054932f, 3.311673164367676f, -11.02491569519043f, 6.5720438957214355f, -13.933245658874512f, 3.847233295440674f, 12.274158477783203f, 0.9272652864456177f, -1.7610269784927368f, 0.4040965139865875f, 0.680636465549469f, -15.860685348510742f, 1.9717633724212646f, -9.28317928314209f, -2.6344847679138184f, 1.870112657546997f, -15.349178314208984f, -5.096085548400879f, -1.4747427701950073f, -8.14248275756836f, 2.2569987773895264f, -15.152320861816406f, -7.701322078704834f, 10.674273490905762f, -1.4894379377365112f, -4.1722493171691895f, -20.385786056518555f, -7.458427429199219f, 0.15474386513233185f, 6.180734634399414f, 9.458477973937988f, 3.9757118225097656f, 3.1847710609436035f, 2.5685551166534424f, -1.0029635429382324f, -0.9310358762741089f, -5.389017581939697f, 13.258286476135254f, 4.855055809020996f, 11.788922309875488f, 2.5168514251708984f, -8.065077781677246f, 5.255640983581543f, 12.24008846282959f, -13.184330940246582f, -0.7134942412376404f, -8.317489624023438f, 11.638908386230469f, -3.975881814956665f, 0.43127647042274475f, -8.266470909118652f, -12.825586318969727f, 13.573546409606934f, 8.546313285827637f, 4.274265289306641f, 3.6718971729278564f, 10.517722129821777f, 0.27425989508628845f, 8.075490951538086f, -2.1264874935150146f, 0.6136637926101685f, -5.506547451019287f, 15.231038093566895f, 0.46134939789772034f, 0.17480090260505676f, 2.040527582168579f, 3.2183902263641357f, -6.139379501342773f, -9.661615371704102f, -10.546405792236328f, -3.0357425212860107f, -3.1846868991851807f, -4.761189937591553f, -3.7215702533721924f, 3.0658512115478516f, -11.569316864013672f, -0.43601199984550476f, 2.8654866218566895f, -2.9958226680755615f, 2.4096181392669678f, -10.564579963684082f, -13.85832405090332f, -6.305012226104736f, 7.627152919769287f, 2.5408518314361572f, -10.277200698852539f, 1.9411969184875488f, -4.243484973907471f, 1.4209707975387573f, 3.253671169281006f, 1.769997000694275f, -9.947007179260254f, 3.4785919189453125f, 3.4994893074035645f, 14.112330436706543f, 17.531051635742188f, 3.3806848526000977f, 2.1774063110351562f, -14.381307601928711f, 6.56845235824585f, -11.52322769165039f, 0.6346222162246704f, 7.065883636474609f, -3.087952136993408f, 6.3684282302856445f, -6.367730140686035f, 6.013090133666992f, 4.485490322113037f, 3.8508636951446533f, -11.656087875366211f, 0.451434463262558f, 7.0519280433654785f, 6.291365623474121f, 10.597298622131348f, 7.171876430511475f, 1.4291812181472778f, -8.897068977355957f, 0.9057914614677429f, 1.817928671836853f, -3.542912006378174f, 15.661563873291016f, -3.5200037956237793f, -1.275672435760498f, 11.505273818969727f, -5.713345527648926f, -13.151139259338379f, 1.984048843383789f, 24.558940887451172f, -0.4125036895275116f, 7.0134735107421875f, 14.921035766601562f, 3.6945953369140625f, 15.289679527282715f, -12.831208229064941f, -7.242607593536377f, -14.355047225952148f, -3.520359992980957f, 17.282093048095703f, -1.093096137046814f, -4.698165416717529f, 4.53480863571167f, -2.2949156761169434f, 2.932422399520874f, 18.38007926940918f, 10.144601821899414f, 1.8887478113174438f, 17.05137825012207f, 2.2348875999450684f, 2.445681571960449f, -16.11703109741211f, 3.6988015174865723f, 28.14395523071289f, -9.436356544494629f, -4.76363468170166f, -9.584181785583496f, -0.4704954922199249f, 1.5917537212371826f, -21.702960968017578f, 6.3507513999938965f, -3.284785270690918f, 7.535861968994141f, 11.226126670837402f, 9.685650825500488f, 0.8183719515800476f, -15.07618236541748f, 6.108888149261475f, 16.537776947021484f, 12.913955688476562f, -4.378338813781738f, 7.780423641204834f, 7.1294660568237305f, 1.4495526552200317f, 0.26958322525024414f, 4.329139709472656f, 6.674724102020264f, -6.520862579345703f, -2.903130531311035f, -17.643451690673828f, 7.825291633605957f, -4.506794452667236f, 0.16557633876800537f, 7.2597246170043945f, 8.140376091003418f, -9.9315185546875f, -6.21274471282959f, -3.006944179534912f, 4.115133285522461f, 1.6233476400375366f, -8.518281936645508f, 1.5647332668304443f, -7.484006881713867f, -4.436017036437988f, 9.502815246582031f, -7.330345630645752f, 7.664921760559082f, -4.133516788482666f, 3.0703797340393066f, 7.408110618591309f, -1.3500370979309082f, -3.113905191421509f, -1.6690822839736938f, 9.191621780395508f, 0.6589356064796448f, -7.368348121643066f, 11.585264205932617f, 2.6871490478515625f, 5.345765113830566f, 1.0851669311523438f, -4.77912712097168f, -9.00320816040039f, -8.105009078979492f, 3.760385274887085f, 8.186501502990723f, 13.72744369506836f, 4.232875823974609f, 4.095627307891846f, 1.6133935451507568f, 14.020605087280273f, -17.89886474609375f, 10.849470138549805f, -4.106210708618164f, -1.5198999643325806f, -4.078025817871094f, 3.1078972816467285f, 6.467252254486084f, 8.850523948669434f, -1.770440936088562f, -3.601609706878662f, 12.48472785949707f, 1.1591122150421143f, 3.630988597869873f, 0.35214078426361084f, 9.08293628692627f, -9.61496353149414f, -3.6545019149780273f, 4.853860378265381f, -21.803064346313477f, 0.9578412175178528f, -5.081964492797852f, -3.8409125804901123f, 13.125861167907715f, 8.514156341552734f, 2.2802765369415283f, -1.5544071197509766f, 7.989774227142334f, -0.9534971117973328f, -20.655189514160156f, 5.728123664855957f, -4.578756809234619f, -9.761033058166504f, -2.411184072494507f, -1.53468918800354f, 1.7299505472183228f, 4.49857759475708f, -5.621171951293945f, -15.699580192565918f, -1.6200847625732422f, 0.2511891722679138f, 12.249534606933594f, -6.566256523132324f, -14.719828605651855f, -11.666041374206543f, 1.057192087173462f, -1.8811215162277222f, -4.319576263427734f, 9.619815826416016f, -6.46962833404541f, -3.5396673679351807f, -4.881474494934082f, -0.6955579519271851f, 3.299593687057495f, 11.470381736755371f, 1.12203049659729f, -0.921830952167511f, 5.743063449859619f, 6.646578311920166f, 1.6719251871109009f, -11.525588989257812f, -12.89569091796875f, 7.118905544281006f, -14.61849594116211f, -9.530622482299805f, -6.113729476928711f, 0.5888143181800842f, -17.381607055664062f, 15.451555252075195f, -6.282611846923828f, -12.82925033569336f, 1.7211908102035522f, -12.8479585647583f, 3.2159461975097656f, 2.13104510307312f, 0.8415284752845764f, -4.35218620300293f, -6.643268585205078f, -3.2159981727600098f, -1.1216778755187988f, -9.839357376098633f, -6.70488977432251f, -4.389339447021484f, -3.2769744396209717f, -5.768002510070801f, -8.732684135437012f, -5.264852046966553f, 6.483272552490234f, 9.102012634277344f, -14.003499984741211f, 0.17809408903121948f, 15.125717163085938f, -12.025708198547363f, -6.3086838722229f, 14.6791353225708f, 9.44178581237793f, -8.969319343566895f, -5.2899250984191895f, -9.268863677978516f, -6.997377872467041f, 9.275823593139648f, 1.1657955646514893f, -7.220368385314941f, 10.789793968200684f, 10.304367065429688f, -4.045272350311279f, -6.108043193817139f, -2.014784812927246f, -11.121515274047852f, -18.38974380493164f, -8.347667694091797f, 5.974706649780273f, 10.243739128112793f, 5.663839340209961f, 8.547972679138184f, -7.52782678604126f, 5.363646507263184f, 15.58993148803711f, 7.026348114013672f, 2.677098512649536f, 18.409914016723633f, -10.548135757446289f, 3.4355015754699707f, 1.4359816312789917f, -2.546203374862671f, -6.266178131103516f, 11.344420433044434f, -19.2717227935791f, -6.462072372436523f, 11.376580238342285f, 3.8701956272125244f, -8.504956245422363f, -7.087599277496338f, -11.178908348083496f, 0.2572647035121918f, 4.125010967254639f, 9.298619270324707f, -8.014323234558105f, 12.535018920898438f, 6.390746116638184f, -6.37876558303833f, 6.679197788238525f, -9.323160171508789f, -5.269445419311523f, 2.596299886703491f, 0.11649034917354584f, 17.804271697998047f, 4.594757556915283f, 6.764013767242432f, 0.6217585206031799f, -7.7316412925720215f, 8.208429336547852f, -7.3180928230285645f, -9.159605979919434f, 8.136429786682129f, -15.975767135620117f, -5.516753673553467f, -7.645376205444336f, -6.816018104553223f, -1.813738226890564f, -0.754987359046936f, 10.540684700012207f, 6.572773456573486f, 4.151686191558838f, 5.905804634094238f, -0.29068470001220703f, -5.357999324798584f, -5.847013473510742f, -4.441774845123291f, 1.8924025297164917f, -1.1913574934005737f, 14.646678924560547f, -13.63921070098877f, 1.4032258987426758f, -2.480900526046753f, 0.2637772262096405f, -7.374143600463867f, 5.216763019561768f, -22.21915626525879f, -1.6103098392486572f, 9.66075325012207f, 7.245777606964111f, -0.4308447539806366f, 3.232591152191162f, 7.705619812011719f, -3.5778160095214844f, -4.7608866691589355f, -3.9943084716796875f, -2.1508443355560303f, 7.583977222442627f, -11.541088104248047f, -9.87782096862793f, -0.3558848798274994f, -0.8333123326301575f, 0.36477258801460266f, -6.956092834472656f, -0.47873327136039734f, -7.288418769836426f, 1.8383797407150269f, -6.37021541595459f, -1.270717740058899f, -5.504342555999756f, -7.638356685638428f, 3.2992103099823f, 3.3208279609680176f, -7.895376682281494f, -0.4653431177139282f, -5.259646415710449f, 0.3640040159225464f, -4.201903820037842f, 7.085029125213623f, 0.7169751524925232f, 1.866424798965454f, 2.98149037361145f, 2.421513319015503f, -4.835546493530273f, -3.400444746017456f, -13.251764297485352f, -7.206017971038818f, 5.5685553550720215f, 8.815764427185059f, 1.3367613554000854f, -12.028512001037598f, -4.593379974365234f, -5.485569000244141f, 7.575770378112793f, 6.186659336090088f, 4.837860107421875f, 4.19438362121582f, -2.0015146732330322f, -8.463085174560547f, 4.390081882476807f, -0.967718243598938f, -6.633697032928467f, 16.146509170532227f, -12.346258163452148f, 5.003246307373047f, 5.257402420043945f, 5.420393943786621f, -4.056494235992432f, -22.63052749633789f, -3.4521031379699707f, 0.5595429539680481f, -0.6451472043991089f, 7.716019630432129f, -3.91341233253479f, 3.537018299102783f, 0.5044690370559692f, -6.045408725738525f, -3.7082343101501465f, 8.852910995483398f, -3.696688175201416f, -2.3492298126220703f, -9.58127212524414f, -12.638714790344238f, 1.8595764636993408f, -4.140509128570557f, -7.7006120681762695f, 1.3192996978759766f, -9.62076473236084f, 9.613175392150879f, 16.121721267700195f, 7.089335918426514f, -11.461114883422852f, -9.27320384979248f, 4.465617656707764f, 1.9801151752471924f, 11.973653793334961f, -9.35599136352539f, 5.719399929046631f, -7.321558475494385f, 17.419307708740234f, -10.423422813415527f, 4.022407531738281f, -1.3151211738586426f, -5.239253044128418f, 0.22252266108989716f, -23.80808448791504f, 11.66386890411377f, -15.182354927062988f, 2.0668842792510986f, -3.095346450805664f, -10.253707885742188f, 4.4772186279296875f, -16.292877197265625f, 2.14457106590271f, 7.256747245788574f, 0.5063409805297852f, -1.0078110694885254f, -17.32522201538086f, -2.448883295059204f, -7.290302753448486f, -10.84791374206543f, 4.010873317718506f, 0.7512247562408447f, -5.4202561378479f, 2.328165292739868f, -6.0993428230285645f, -12.868612289428711f, 4.567487716674805f, -10.598518371582031f, -11.058136940002441f, -1.659664273262024f, 10.452244758605957f, 18.823354721069336f, -8.08329963684082f, 4.401593208312988f, 2.138773202896118f, -13.085124969482422f, 2.258606195449829f, -2.7691009044647217f, 5.584531784057617f, 10.423367500305176f, 2.846891164779663f, -4.120229721069336f, 1.5436748266220093f, -13.3086519241333f, -8.132325172424316f, 13.937261581420898f, -14.859661102294922f, -10.103026390075684f, 10.011719703674316f, -0.9183381795883179f, 9.600658416748047f, -9.821544647216797f, -9.450371742248535f, -2.242305278778076f, -6.559144020080566f, -4.224136829376221f, -11.820085525512695f, -6.345705986022949f, 8.94025993347168f, 4.509256839752197f, -2.219336748123169f, -0.8802061080932617f, -14.329898834228516f, 7.088708877563477f, -9.526060104370117f, 8.278675079345703f, 2.4871132373809814f, -9.620868682861328f, 9.28262710571289f, 14.8040132522583f, -19.852964401245117f, 4.130612850189209f, 5.440346717834473f, 7.126719951629639f, 0.05489916726946831f, -1.4817479848861694f, 6.829895973205566f, 12.818050384521484f, 9.886153221130371f, 1.254286766052246f, -8.354349136352539f, -11.958121299743652f, 9.16726016998291f, 4.949143886566162f, -7.31182861328125f, 18.494693756103516f, -9.817893028259277f, -0.3160404562950134f, -7.171751022338867f, 15.233163833618164f, -2.146108388900757f, 5.443892478942871f, 4.517480850219727f, 14.043611526489258f, 10.903755187988281f, -7.276175022125244f, 13.366379737854004f, -8.15054702758789f, 0.6368334293365479f, 9.139420509338379f, -6.253448963165283f, 7.538908958435059f, -4.911654949188232f, 1.2663601636886597f, 5.10722017288208f, -10.726715087890625f, -9.355097770690918f, -0.3182983100414276f, -0.6880562901496887f, 1.4052690267562866f, 8.107300758361816f, -17.92100715637207f, -4.534452438354492f, 0.5386764407157898f, 12.413888931274414f, -7.201480388641357f, -13.179725646972656f, -8.37654972076416f, -0.3154694736003876f, 3.091693878173828f, -1.0480709075927734f, 3.751927614212036f, 5.383627414703369f, 6.276157855987549f, 5.926423072814941f, -12.69626235961914f, 0.419791042804718f, 5.777501583099365f, -2.8398115634918213f, -4.607639789581299f, 6.945610523223877f, -3.02091383934021f, -9.469738006591797f, -7.620004177093506f, -2.214186191558838f, -10.362244606018066f, -3.8059444427490234f, -8.182043075561523f, 12.854665756225586f, -9.528903007507324f, 0.49294862151145935f, -8.579874038696289f, 2.118196725845337f, 7.62219762802124f, -5.029037952423096f, 12.290265083312988f, 0.5394968390464783f, 0.8183786869049072f, 1.3815820217132568f, -8.656240463256836f, -10.229368209838867f, 5.616023540496826f, 1.3402541875839233f, 0.14747254550457f, 5.180647373199463f, -5.916377544403076f, 11.535149574279785f, 6.629460334777832f, -2.8352251052856445f, 2.0122580528259277f, -0.12555190920829773f, 0.09699758142232895f, -3.735365867614746f, 6.413780212402344f, -9.827960968017578f, 4.477472305297852f, 3.9674675464630127f, 13.4703950881958f, -10.233613014221191f, 6.505621433258057f, -0.8457767963409424f, 2.9881680011749268f, -1.2331901788711548f, 0.2708655893802643f, -20.598543167114258f, -3.6357667446136475f, -10.855840682983398f, 1.586439847946167f, 10.175971031188965f, -1.9043898582458496f, 2.3536105155944824f, 4.586009502410889f, -16.701967239379883f, -5.804449558258057f, -2.78130841255188f, -12.884259223937988f, 4.541162967681885f, -2.2364847660064697f, 7.2929511070251465f, -20.930879592895508f, -1.1628355979919434f, -0.6431397795677185f, -4.187624454498291f, 9.130221366882324f, 5.2739481925964355f, -5.0708441734313965f, 0.17360350489616394f, 0.5822219252586365f, -0.22339344024658203f, -2.761117458343506f, -10.778762817382812f, 5.385908603668213f, -7.857998371124268f, 2.606891393661499f, 18.175046920776367f, -2.693230390548706f, 10.186522483825684f, -10.02297306060791f, -2.4003751277923584f, 4.620128631591797f, 11.451003074645996f, -9.571845054626465f, -7.237552165985107f, 16.154325485229492f, -19.152278900146484f, 0.16639988124370575f, -1.1797699928283691f, -3.471789836883545f, 7.653792858123779f, 3.1148579120635986f, -17.236486434936523f, 7.4174604415893555f, 5.9619293212890625f, 3.140089511871338f, -1.4843370914459229f, -11.55695629119873f, 4.42191219329834f, -1.2158066034317017f, 1.4187449216842651f, -4.891499042510986f, -0.7901540398597717f, 3.7802929878234863f, -5.735926151275635f, -6.485677242279053f, -1.5401825904846191f, 0.4863590598106384f, 7.93129301071167f, -1.8886117935180664f, 13.101932525634766f, -9.918781280517578f, 7.965061664581299f, -7.907785892486572f, -1.5377079248428345f, 13.429678916931152f, -9.150714874267578f, 11.128018379211426f, 14.368767738342285f, 3.5130884647369385f, -20.24823570251465f, 5.9857892990112305f, -16.631755828857422f, 6.948392391204834f, -0.7420387864112854f, 13.481123924255371f, -6.017175674438477f, -6.115068435668945f, -3.0654659271240234f, 13.489151000976562f, -2.2497708797454834f, 15.904927253723145f, -5.288370132446289f, -2.09615421295166f, -1.6209585666656494f, -10.90034008026123f, -2.202252149581909f, -7.51241397857666f, 1.12482488155365f, -3.7093164920806885f, 5.637340068817139f, 3.773543357849121f, 1.558279275894165f, 0.7900522947311401f, 11.120495796203613f, -4.0744709968566895f, 10.195045471191406f, -1.018953561782837f, 15.167667388916016f, 2.601929187774658f, 0.008113066665828228f, -9.717181205749512f, 6.3429789543151855f, 21.926603317260742f, -9.98530387878418f, 14.128251075744629f, 6.466258525848389f, -6.853448390960693f, -4.719650745391846f, -11.167947769165039f, -3.2867848873138428f, -12.95023250579834f, -5.806682109832764f, 11.830602645874023f, -5.1555986404418945f, -6.032625198364258f, -7.19350528717041f, -0.5742126703262329f, -20.182292938232422f, -2.5560240745544434f, 8.388191223144531f, 3.965162754058838f, -2.2216339111328125f, -11.404376983642578f, 15.09308910369873f, 8.153555870056152f, -3.3941447734832764f, -5.3842549324035645f, 17.98311996459961f, -0.8321418166160583f, -13.44106388092041f, -20.923404693603516f, -7.57475471496582f, -3.0816946029663086f, 14.614544868469238f, 3.570502519607544f, 4.2990217208862305f, 0.1919950395822525f, 1.0185343027114868f, -8.14962387084961f, 10.906466484069824f, 15.584993362426758f, -0.4128416180610657f, -6.817087173461914f, 17.120927810668945f, 2.2402291297912598f, -1.9622392654418945f, -3.413076162338257f, 6.840425968170166f, -5.948558330535889f, 6.701462745666504f, 0.8215048909187317f, -1.0856260061264038f, -3.9645702838897705f, 1.4618407487869263f, -1.7632997035980225f, 0.5018937587738037f, -2.7499217987060547f, 4.106265068054199f, -12.30443000793457f, 6.145236492156982f, 4.5076003074646f, 3.7207319736480713f, 9.945277214050293f, 1.5612727403640747f, -25.549476623535156f, -4.634391784667969f, -9.542594909667969f, 10.549148559570312f, 2.471867322921753f, 1.1155062913894653f, -2.9077959060668945f, 7.583897590637207f, -2.4130048751831055f, 8.161357879638672f, -6.786379337310791f, 15.053579330444336f, 2.3771886825561523f, 8.431512832641602f, -10.34504508972168f, 3.9834461212158203f, 0.3579562306404114f, -14.247590065002441f, -11.214945793151855f, -6.256213188171387f, 8.011175155639648f, 9.249755859375f, -11.789390563964844f, -14.64880657196045f, -0.27090367674827576f, -0.2744891941547394f, 11.454398155212402f, -14.925914764404297f, -7.099286079406738f, -3.3692843914031982f, 6.4099931716918945f, 6.6391777992248535f, 10.544389724731445f, 3.6568782329559326f, 6.840872764587402f, 12.177023887634277f, 3.4128663539886475f, -3.8373212814331055f, -0.12041311711072922f, -13.457633972167969f, 18.881927490234375f, 8.021130561828613f, -7.102904796600342f, 7.734999179840088f, 3.826378107070923f, -0.7286080718040466f, -6.966921329498291f, -9.949929237365723f, -6.5738348960876465f, -6.720236778259277f, -2.4046998023986816f, -7.772119998931885f, -3.3762123584747314f, 16.03032875061035f, -10.771323204040527f, -0.8427897095680237f, 2.035958766937256f, -9.551191329956055f, 13.150819778442383f, -8.132336616516113f, -0.9750276803970337f, 2.9933242797851562f, -12.041450500488281f, -7.16148042678833f, -7.287316799163818f, 5.334017753601074f, -7.423707962036133f, -7.503839015960693f, -13.0750150680542f, -1.5110211372375488f, -3.3898580074310303f, 6.609547138214111f, -1.1499853134155273f, 6.6649370193481445f, -4.917843341827393f, -8.266522407531738f, 16.943111419677734f, 0.35688164830207825f, 2.3017640113830566f, -2.517993927001953f, -20.720897674560547f, -5.1919379234313965f, 6.862222194671631f, 8.344823837280273f, -3.240790367126465f, -0.7951709032058716f, -7.633636951446533f, 1.9410372972488403f, -3.9540672302246094f, -8.897167205810547f, 9.550298690795898f, -12.049423217773438f, 3.2460708618164062f, -0.7465460896492004f, -7.343610763549805f, -25.89638328552246f, 1.305576205253601f, 0.7940993309020996f, 15.786218643188477f, -5.214616298675537f, 5.427506923675537f, -9.456047058105469f, -4.363546371459961f, 14.75342845916748f, -8.062644958496094f, 12.066773414611816f, -17.939416885375977f, -0.6330687403678894f, -8.203537940979004f, -15.027321815490723f, -7.841004848480225f, -6.411040306091309f, -22.517040252685547f, -4.999447822570801f, -8.532695770263672f, 8.279166221618652f, 7.497697830200195f, 8.378273963928223f, -2.62188458442688f, -3.915342330932617f, 3.5234627723693848f, -6.344028949737549f, 7.4199934005737305f, -4.750202655792236f, 1.74264395236969f, 8.607345581054688f, -10.318155288696289f, -0.5510790944099426f, 24.487709045410156f, -4.434352397918701f, 3.7080373764038086f, -16.31922149658203f, 2.130758285522461f, -1.8111369609832764f, 16.641450881958008f, -15.62347412109375f, 9.540854454040527f, -3.6384425163269043f, 1.0087566375732422f, 2.4287056922912598f, 1.3515430688858032f, -1.9539055824279785f, 7.968528747558594f, 3.1463985443115234f, 4.322893142700195f, -4.588891506195068f, 7.193646430969238f, 3.6794357299804688f, -3.6867868900299072f, -12.593381881713867f, 22.773561477661133f, -2.592869997024536f, -8.03166389465332f, 1.6411826610565186f, -5.852877140045166f, 6.320628643035889f, -1.24896240234375f, 2.633263349533081f, 2.3571205139160156f, 2.6310131549835205f, -4.19133996963501f, 9.353876113891602f, -15.862900733947754f, -11.460186958312988f, -0.3524762690067291f, 3.5821545124053955f, -4.435649871826172f, -1.1791728734970093f, 2.729346990585327f, -1.2911462783813477f, -0.1667870581150055f, 3.152031421661377f, -20.307052612304688f, -16.968524932861328f, -2.394564628601074f, -4.503590106964111f, 3.181205987930298f, 0.05041913315653801f, -0.3948255181312561f, -6.3789544105529785f, 1.732195258140564f, -5.6798787117004395f, 7.7667107582092285f, 9.2898530960083f, -4.316888332366943f, -16.857954025268555f, -5.379256725311279f, 4.286224842071533f, 4.1023759841918945f, -8.238680839538574f, 9.628629684448242f, -2.3275868892669678f, 2.8554039001464844f, -8.514399528503418f, -9.410696029663086f, 4.0246405601501465f, -6.667844295501709f, -5.36508321762085f, 0.8656852841377258f, 2.9765312671661377f, -0.6338213682174683f, -4.245971202850342f, 6.7638702392578125f, 8.446202278137207f, -2.508578300476074f, -0.5704231858253479f, 0.46178925037384033f, 10.725545883178711f, 4.313973426818848f, -6.148879528045654f, 3.38698410987854f, -12.07677173614502f, 8.890965461730957f, 9.622424125671387f, -1.9190295934677124f, 1.035591721534729f, 3.041085720062256f, 11.527141571044922f, 2.2250008583068848f, 0.06786001473665237f, 11.618230819702148f, 0.2951813340187073f, 7.107181549072266f, -13.101400375366211f, -4.043383598327637f, 5.551668643951416f, 0.23836034536361694f, -9.364093780517578f, 0.38196349143981934f, 5.709733486175537f, -3.463144540786743f, 10.42895793914795f, -21.435224533081055f, -22.471851348876953f, 2.9046216011047363f, 5.418637275695801f, -2.009808301925659f, -12.568074226379395f, 5.197458744049072f, 12.429717063903809f, -3.8452060222625732f, -8.332062721252441f, -1.3275021314620972f, 8.634135246276855f, -17.408498764038086f, 4.173489570617676f, 3.602097272872925f, 16.919105529785156f, -14.429887771606445f, -7.9446563720703125f, -15.02901554107666f, 5.9782915115356445f, -2.348747491836548f, 2.622194766998291f, 9.501197814941406f, 6.67450475692749f, 5.712249279022217f, -1.8605291843414307f, -0.874760627746582f, 15.388128280639648f, 2.725569486618042f, -4.520296096801758f, -11.915221214294434f, 3.2612195014953613f, -10.38715934753418f, 2.1763875484466553f, -1.2297403812408447f, 6.042415618896484f, -13.808868408203125f, -5.520578861236572f, -1.4444622993469238f, -7.977278232574463f, 8.31882381439209f, -9.937228202819824f, 7.792935848236084f, -8.416111946105957f, 7.689346790313721f, -4.904921054840088f, -0.9467958211898804f, -2.2417097091674805f, 4.855114459991455f, 12.061532020568848f, -4.05377721786499f, 7.5981645584106445f, -5.726215362548828f, -5.700634956359863f, -3.6235997676849365f, -8.09415340423584f, -8.389449119567871f, 3.4986400604248047f, -0.17018863558769226f, -4.8295674324035645f, 3.72519588470459f, 6.054676532745361f, 2.607694149017334f, -5.292807579040527f, 10.3839111328125f, 2.9594953060150146f, 4.974825382232666f, 30.000545501708984f, -6.096185207366943f, 0.5663076043128967f, 4.246780872344971f, -7.833420753479004f, -6.573525905609131f, -4.979223251342773f, 9.320971488952637f, 2.075157642364502f, 0.24154403805732727f, 13.304895401000977f, -9.484021186828613f, 8.853174209594727f, -7.756874084472656f, 0.7791877388954163f, 13.578595161437988f, -4.487305641174316f, -1.5067740678787231f, -20.156299591064453f, 0.3791326582431793f, -4.492989540100098f, 11.668903350830078f, -3.7837438583374023f, -0.9617695808410645f, -15.763601303100586f, 12.063730239868164f, 0.8532925248146057f, -9.071452140808105f, 3.879969358444214f, 14.453145027160645f, -2.9419867992401123f, 8.816530227661133f, 6.016716957092285f, -15.384603500366211f, 7.9487481117248535f, 5.712857723236084f, 6.595734596252441f, -8.348841667175293f, -7.271892070770264f, 8.39649772644043f, 3.010307550430298f, -7.354966640472412f, 4.326541900634766f, 6.269808292388916f, -2.21332049369812f, 2.8673322200775146f, -11.210542678833008f, 8.31998062133789f, -12.9059419631958f, 8.617777824401855f, -3.5901377201080322f, -10.140533447265625f, 0.8802559971809387f, -9.837919235229492f, -9.109603881835938f, -15.81827449798584f, 4.78360652923584f, -1.1366561651229858f, -22.160612106323242f, -12.40213680267334f, -18.063514709472656f, -2.445664882659912f, 15.82851505279541f, -4.466148853302002f, 7.4260993003845215f, 10.562188148498535f, 2.9546096324920654f, 6.616511821746826f, -11.446455001831055f, 1.037653923034668f, 8.754400253295898f, 9.139323234558105f, -6.899957180023193f, -8.12019157409668f, 10.253373146057129f, 3.2580606937408447f, -1.1452258825302124f, 6.390077114105225f, -1.9534237384796143f, -3.5342469215393066f, -6.437942981719971f, -11.188612937927246f, 1.4919053316116333f, 12.896910667419434f, 4.564462184906006f, -8.683686256408691f, 2.0561792850494385f, -7.927750587463379f, -7.966180324554443f, 3.0324785709381104f, -14.164445877075195f, -0.5540791153907776f, 7.670888900756836f, -0.11896628886461258f, 1.1451127529144287f, 22.234376907348633f, -5.0322980880737305f, 10.48377513885498f, 10.193860054016113f, -6.44713020324707f, 4.20560359954834f, -19.378570556640625f, 7.226066589355469f, -11.02578067779541f, 16.957338333129883f, 5.088015079498291f, 7.4178314208984375f, 0.37720149755477905f, 17.275354385375977f, 5.082028865814209f, 0.11316994577646255f, 10.450178146362305f}; - -// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel). -// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad -// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs. -// quad ptr for (m_tile mt, k_group g) = Apack_dram + (mt*(K/4)+g)*64. -static float Apack_dram[64*64] __attribute__((section(".data"))) = -{1.9269152879714966f, 1.4451338052749634f, 1.931160569190979f, 1.7743884325027466f, -0.685481607913971f, 0.4334380030632019f, 0.6663737893104553f, -1.3838845491409302f, -1.363769769668579f, -0.8032152652740479f, -1.2786966562271118f, 0.3220294117927551f, 0.34436315298080444f, -0.22056575119495392f, -0.7111086845397949f, -0.447700172662735f, 1.4872840642929077f, 0.8564125299453735f, 1.0118638277053833f, -0.921550989151001f, 0.5635589957237244f, -0.7171905636787415f, -0.07426456362009048f, 0.4868715703487396f, 0.1929578185081482f, -1.1209005117416382f, -0.03842746838927269f, 0.44605663418769836f, -3.1016058921813965f, 0.7118127346038818f, -0.3866667151451111f, -0.7288169264793396f, 0.9007171988487244f, 2.218075752258301f, -1.4364064931869507f, 0.9624499082565308f, -1.507175087928772f, 1.055369257926941f, -0.20960482954978943f, -1.0020296573638916f, -0.6103342771530151f, 0.1956406533718109f, 1.9138009548187256f, 1.5229592323303223f, -1.4587225914001465f, 0.3415871560573578f, 0.9578314423561096f, -0.16066236793994904f, -2.1055209636688232f, 0.5231655240058899f, -1.1298598051071167f, -0.33701515197753906f, -1.610666036605835f, -1.4533969163894653f, 0.16632132232189178f, 0.0329488180577755f, 0.16323445737361908f, -0.7815181016921997f, 0.3378446102142334f, 1.2804899215698242f, -1.4318257570266724f, 1.5885895490646362f, -0.8225308656692505f, -0.32063713669776917f, 0.6784184575080872f, 0.34664666652679443f, -0.1360345333814621f, -1.1753336191177368f, -1.4790465831756592f, 0.46515071392059326f, 1.4703037738800049f, -0.42919591069221497f, 1.51017165184021f, -1.7898789644241333f, 0.12505611777305603f, -0.11616043001413345f, -0.6071268916130066f, -0.3488781750202179f, -2.3908050060272217f, -0.6307737827301025f, -1.2345448732376099f, -0.19733144342899323f, 1.6354095935821533f, 0.35805708169937134f, 0.4322742819786072f, 0.37139150500297546f, -0.9390866756439209f, -0.9817978739738464f, 0.21230429410934448f, -0.26157355308532715f, -0.7621514797210693f, 1.3705363273620605f, -0.25973787903785706f, -0.45791950821876526f, 0.322247713804245f, -0.788766622543335f, -0.04306747764348984f, -1.0545889139175415f, 0.6547407507896423f, 0.47876790165901184f, -0.1250254064798355f, -0.004656785633414984f, -0.6013189554214478f, -0.6420586109161377f, -0.7252011299133301f, -0.44025033712387085f, -1.190559983253479f, -0.4809381365776062f, -0.7190185785293579f, -1.2322070598602295f, 1.875388741493225f, 1.3061575889587402f, -1.6046669483184814f, 1.2779955863952637f, 0.5760045647621155f, 1.353700041770935f, 0.7821183800697327f, 0.07954943925142288f, -0.09964022785425186f, 0.8265887498855591f, -0.9527732729911804f, 2.1848294734954834f, 0.7756073474884033f, -0.9903622269630432f, -0.38583096861839294f, -0.598077118396759f, 1.1042989492416382f, -0.9275763630867004f, -0.7521352767944336f, -0.1721901297569275f, 1.1415079832077026f, 0.5260620713233948f, -1.598767638206482f, 0.3781784772872925f, -0.9851518273353577f, 1.591395616531372f, 0.5216943025588989f, -0.48009708523750305f, 0.455719918012619f, -1.3641812801361084f, 0.5233525037765503f, -0.28154700994491577f, -0.5223758816719055f, -0.26273947954177856f, 1.6487230062484741f, 0.5237884521484375f, 0.018564576283097267f, 2.1120378971099854f, -0.10912995040416718f, 0.7051141262054443f, -2.488459348678589f, -0.1208132952451706f, -0.46386733651161194f, -1.2871733903884888f, 0.2503303289413452f, 0.008205652236938477f, -0.8211768269538879f, 0.05281926319003105f, -0.7401803731918335f, 0.9314952492713928f, -0.3924786448478699f, 0.056621819734573364f, -1.8058050870895386f, -0.5207571387290955f, 0.7151994705200195f, -1.7236974239349365f, -0.33131900429725647f, -0.48302069306373596f, 0.18237744271755219f, 0.7388824224472046f, -1.3610970973968506f, -0.40586018562316895f, -0.47086891531944275f, 0.42497751116752625f, 0.16235657036304474f, -0.4593467116355896f, -1.4036071300506592f, 0.4262961447238922f, 0.9254348874092102f, -0.9320061206817627f, 0.03913922235369682f, -0.8434810638427734f, 0.8435799479484558f, 0.11329790204763412f, -0.38666075468063354f, 0.03389474004507065f, 1.8018341064453125f, -0.7110859751701355f, 0.6016423106193542f, 0.4825834333896637f, -0.2369976043701172f, -0.9419456720352173f, -0.7278813123703003f, 0.575005054473877f, -0.3753443658351898f, 0.18516133725643158f, 1.305860161781311f, 0.4351435601711273f, 0.9874473810195923f, 0.0771508663892746f, -1.7906768321990967f, -0.31229403614997864f, -0.07434194535017014f, -0.3495793640613556f, -0.28251126408576965f, 0.48813387751579285f, 0.5099347233772278f, -0.7089186310768127f, -0.5594301819801331f, -0.6417241096496582f, 1.0330873727798462f, 1.0686918497085571f, 0.24659274518489838f, 0.26588720083236694f, -0.33197471499443054f, -0.9228128790855408f, 0.09329313784837723f, -0.2541753351688385f, -0.15664155781269073f, 0.3797488212585449f, 0.7692679762840271f, 1.0082393884658813f, 1.670624852180481f, 2.1860759258270264f, -0.7688388824462891f, -2.2063984870910645f, -0.6866509318351746f, 1.3065344095230103f, -1.9775909185409546f, -0.5870985388755798f, -0.8076189756393433f, -1.2619991302490234f, -1.9152568578720093f, -1.205536127090454f, -0.8708454966545105f, 0.9993040561676025f, -0.7668924331665039f, -0.595004141330719f, 1.5921050310134888f, -0.6493158936500549f, 0.7624453902244568f, -0.7508030533790588f, 0.6368136405944824f, 0.4598345160484314f, 0.01789604313671589f, 0.0826888456940651f, 0.824364185333252f, 1.0860532522201538f, -0.6421752572059631f, -0.9542103409767151f, -0.6410972476005554f, 1.2751853466033936f, -0.9494866728782654f, 0.3926331400871277f, -0.41619211435317993f, 0.45214036107063293f, 1.6423169374465942f, 0.01086814422160387f, -0.9726738929748535f, -0.8146268725395203f, -1.3793021440505981f, 0.8853808045387268f, 0.024699924513697624f, 1.096641182899475f, 1.3438509702682495f, 0.061276569962501526f, -0.414562851190567f, 0.9594927430152893f, 0.01691739819943905f, 0.8229668736457825f, 1.861944556236267f, 0.8520749807357788f, -0.1595974713563919f, -0.33874234557151794f, 0.9584577679634094f, -1.0212392807006836f, 0.625802755355835f, 0.1824439913034439f, -1.0641486644744873f, -0.6836934685707092f, -1.2922308444976807f, 0.08526104688644409f, -0.6902380585670471f, 0.10350999981164932f, 0.08027740567922592f, -0.886031985282898f, -1.077892780303955f, -1.6946725845336914f, -0.4973975419998169f, -1.3406795263290405f, 1.6192004680633545f, -0.49492356181144714f, -2.5849502086639404f, 0.7863810062408447f, -0.7601934671401978f, 0.06604336202144623f, 0.766244113445282f, 0.7481252551078796f, -0.22995619475841522f, 0.8290349841117859f, 0.7448412775993347f, 1.4801039695739746f, 0.8848565220832825f, 1.1805996894836426f, 0.439589262008667f, -0.5853705406188965f, 1.450609803199768f, -0.5922516584396362f, -0.02399955503642559f, -0.057920295745134354f, -0.4075061082839966f, -0.0007737990817986429f, 0.64540034532547f, -0.16356196999549866f, -2.172283887863159f, 2.0921294689178467f, 1.345484972000122f, 0.8391514420509338f, -0.8342104554176331f, -2.8929238319396973f, -0.7581311464309692f, 0.5361881256103516f, 0.2694815397262573f, 0.15431594848632812f, -0.1221928745508194f, 0.5666652917861938f, 0.9623646140098572f, 0.1620604395866394f, 0.353316068649292f, -0.9085567593574524f, 0.08768323808908463f, 0.7953095436096191f, 0.12682189047336578f, -0.20004984736442566f, 1.0300744771957397f, -0.3875778615474701f, 1.078317642211914f, 0.5246226191520691f, -0.21037597954273224f, 0.4407670795917511f, -0.7469954490661621f, -0.7097623348236084f, -0.14264194667339325f, 1.195958137512207f, -2.6474881172180176f, 0.3129958212375641f, 1.0937845706939697f, 0.2792847752571106f, -2.4520716667175293f, 0.9949536919593811f, -0.8680985569953918f, -0.7124031782150269f, 0.8008005619049072f, 1.1412016153335571f, -0.7328027486801147f, -0.14829230308532715f, 1.7093087434768677f, -0.4875054359436035f, 0.15271379053592682f, -1.3061535358428955f, -1.4575366973876953f, 0.8050477504730225f, -0.1177205815911293f, 0.1864478439092636f, 0.4159761369228363f, 0.7201864719390869f, -0.5701602697372437f, -1.6171332597732544f, 1.680620551109314f, 0.05164359509944916f, 0.10429783165454865f, -2.3184432983398438f, 0.05792269483208656f, 0.050095997750759125f, -0.0388023778796196f, -1.4039719104766846f, -0.9712379574775696f, -1.1133604049682617f, -0.29864323139190674f, 0.3547132909297943f, 1.9025356769561768f, -0.13413065671920776f, 0.32332202792167664f, -0.35899198055267334f, 1.27912437915802f, 0.7439519762992859f, 0.3487516939640045f, -0.39799532294273376f, 1.1929805278778076f, 0.6084084510803223f, 0.9446058869361877f, -1.0597201585769653f, 0.25403109192848206f, 0.4981626570224762f, -0.9536206126213074f, 0.09063850343227386f, -0.7346699833869934f, -1.4067999124526978f, 1.1284750699996948f, 0.051366694271564484f, 1.2964228391647339f, -0.4815843999385834f, 0.9675941467285156f, 1.0804862976074219f, 1.9372931718826294f, 1.6308681964874268f, -1.5824053287506104f, 0.3057299852371216f, -0.1790592074394226f, -1.1999552249908447f, -0.09247277677059174f, 1.7422553300857544f, 0.044657133519649506f, -2.3609628677368164f, -1.2123126983642578f, 0.6950237154960632f, 0.610466480255127f, -1.0494661331176758f, -0.46568843722343445f, -1.7808643579483032f, 0.7287133932113647f, -0.08472306281328201f, 0.9871290922164917f, 0.4150581359863281f, 1.1992844343185425f, 0.12711313366889954f, -1.01665461063385f, -1.2660012245178223f, -1.5211198329925537f, -0.2904934287071228f, 2.602391004562378f, 1.835181474685669f, 1.334737777709961f, 0.603898823261261f, 1.6047972440719604f, 1.5080454349517822f, 0.9808937907218933f, 1.0844124555587769f, 1.1456739902496338f, -0.7174144983291626f, -0.4292171895503998f, 0.4403660297393799f, -0.007675689645111561f, 0.38916081190109253f, 0.3478375971317291f, -0.13345853984355927f, -0.09572362899780273f, -1.9180361032485962f, -0.2316243201494217f, -1.7222950458526611f, -2.4801201820373535f, 0.30942854285240173f, 0.41459226608276367f, 0.9477656483650208f, -0.14181147515773773f, 2.833967924118042f, 1.010284185409546f, 0.6377718448638916f, -0.518220841884613f, 0.34287506341934204f, 0.7401772737503052f, -0.15693345665931702f, -0.08114803582429886f, -1.3923954963684082f, 0.041759490966796875f, -0.827768862247467f, -0.4175437390804291f, -0.5003090500831604f, 1.15656316280365f, -0.676629364490509f, -0.2763414680957794f, 1.9534740447998047f, 0.6110401153564453f, 0.15978877246379852f, 0.83954256772995f, -1.4590637683868408f, 1.4161995649337769f, 1.138344645500183f, 1.2586976289749146f, 0.540465772151947f, -0.2515752911567688f, 1.334702968597412f, -1.1954537630081177f, 1.0350031852722168f, 0.2690545618534088f, -0.5730168223381042f, -0.19321373105049133f, 2.0486814975738525f, 1.2208385467529297f, 1.7697970867156982f, 0.05852266773581505f, -1.4936561584472656f, 0.6833979487419128f, -0.2505214214324951f, 0.8691263794898987f, 0.4350730776786804f, 0.859858512878418f, 0.48353928327560425f, 0.8123369216918945f, 1.6896470785140991f, -0.036629438400268555f, -0.3303174376487732f, 0.7767809629440308f, -1.0880382061004639f, -0.6076440215110779f, 0.6268176436424255f, -1.6682480573654175f, -0.22138521075248718f, -0.13825182616710663f, 1.6704555749893188f, -0.9609367251396179f, -2.2717032432556152f, -1.3846737146377563f, -2.5095443725585938f, -1.9005532264709473f, -0.004505051765590906f, 0.9732939004898071f, -0.7939430475234985f, 0.6838752627372742f, 1.621694564819336f, -1.7376028299331665f, -1.8736529350280762f, 2.129624843597412f, 0.22523505985736847f, 0.9212995171546936f, -0.545271098613739f, 0.05182264745235443f, -0.13386189937591553f, -0.8712361454963684f, 0.4880010485649109f, 0.22857652604579926f, 1.666792392730713f, -1.0150787830352783f, 0.3752319812774658f, -1.3245893716812134f, 0.8512656688690186f, -0.12535162270069122f, 2.3259060382843018f, -1.5181471109390259f, -0.07724537700414658f, 0.5282443761825562f, -2.15816330909729f, -0.3284812867641449f, -0.058557309210300446f, -0.223365917801857f, 0.7845868468284607f, 0.02485940419137478f, 0.15392017364501953f, -0.5419175624847412f, 0.08790969103574753f, -0.5160817503929138f, -0.40046969056129456f, -1.3658148050308228f, -0.9203909635543823f, 0.1387282907962799f, 0.9856945276260376f, -0.008228386752307415f, -1.6607975959777832f, -2.247206211090088f, 0.12574470043182373f, 1.7173614501953125f, 0.02864718623459339f, -0.34595024585723877f, -1.0602530241012573f, -0.44102486968040466f, -1.241483449935913f, 0.6001842617988586f, -0.6088271737098694f, 1.111746072769165f, 0.6661149263381958f, -1.1797568798065186f, 1.2783364057540894f, -1.4493319988250732f, -0.6637441515922546f, -0.4478967487812042f, -0.5525766611099243f, 0.3188803195953369f, 0.640755295753479f, 0.2868328094482422f, -0.572657585144043f, -0.3136177957057953f, -0.32025346159935f, -0.4702208340167999f, -0.508095383644104f, -0.6227966547012329f, -0.44026491045951843f, -0.5297411680221558f, 0.28815189003944397f, -0.605182409286499f, 0.3657907545566559f, 0.4234687089920044f, 0.07448001205921173f, -0.42451897263526917f, 0.5832474231719971f, -0.7308424115180969f, 0.0835680365562439f, -0.12925422191619873f, -0.844377875328064f, -0.6086435317993164f, -0.6184902191162109f, -0.7891808748245239f, -2.3179564476013184f, 0.9625157713890076f, 0.869049608707428f, -0.17924511432647705f, -0.39920157194137573f, -0.3874586224555969f, -0.1492866724729538f, 0.3057209253311157f, 1.0669267177581787f, 0.17482025921344757f, 0.39990535378456116f, -0.7149624228477478f, -0.5513465404510498f, -0.046192023903131485f, -1.647040605545044f, -0.167823925614357f, 1.294582724571228f, 0.2794382870197296f, -0.8097057938575745f, 0.19955870509147644f, 0.49674081802368164f, -0.22963792085647583f, -0.5522539615631104f, -0.7745925188064575f, -0.4501533806324005f, -1.0939292907714844f, 1.989207148551941f, -0.047562163323163986f, 1.9889612197875977f, -1.6457397937774658f, -1.0362098217010498f, 1.6433145999908447f, 0.22267311811447144f, -0.5718191266059875f, -1.4298604726791382f, -1.2461947202682495f, -2.369169235229492f, -0.40709349513053894f, -0.09342008084058762f, -1.5575724840164185f, -0.18526747822761536f, -1.6021603345870972f, -0.07198750972747803f, 2.0207436084747314f, 1.900311827659607f, -0.4833274185657501f, -0.4503057301044464f, 2.0070879459381104f, -0.8483412265777588f, -2.7936289310455322f, 0.45901596546173096f, -0.41459938883781433f, -0.5614708065986633f, 0.8702965974807739f, -1.0284309387207031f, 0.9956361055374146f, 0.7527588605880737f, 1.3528969287872314f, -0.906094491481781f, 0.25391900539398193f, 1.6950805187225342f, -0.740294337272644f, -0.0729660615324974f, -1.2531019449234009f, 1.6489421129226685f, -0.7111545205116272f, 0.5309328436851501f, 1.4558700323104858f, -0.5949130654335022f, -1.0552809238433838f, 0.40444278717041016f, -0.8797858357429504f, 0.4047577977180481f, 1.288827657699585f, -2.0487122535705566f, 0.9364385008811951f, 0.028089528903365135f, 0.31428107619285583f, -0.5479549169540405f, 1.118869423866272f, 1.6005686521530151f, 0.5235219597816467f, -1.3614802360534668f, 0.3316534161567688f, 1.2687278985977173f, -1.3284013271331787f, 2.1425962448120117f, -0.6011420488357544f, 0.17846599221229553f, 0.05229547247290611f, -1.0810555219650269f, 0.7122363448143005f, -0.17536965012550354f, 0.1415553092956543f, -1.1425532102584839f, 1.7732776403427124f, -0.07858924567699432f, -1.71055006980896f, 1.9562491178512573f, -1.00010085105896f, 1.2904434204101562f, 0.7060741186141968f, -0.5153723955154419f, -1.2741512060165405f, 0.2649095058441162f, -1.5468504428863525f, 0.01762307994067669f, -0.031765542924404144f, -1.7734959125518799f, 1.0348176956176758f, -0.44875210523605347f, -2.071660280227661f, 0.43104586005210876f, 0.8384853601455688f, 1.7684898376464844f, -0.6919524669647217f, -1.1755682229995728f, 0.35730111598968506f, 1.0827196836471558f, 2.1227850914001465f, 1.2731683254241943f, 0.7567060589790344f, 0.0782259851694107f, 0.10164086520671844f, -0.7046414017677307f, -0.626437783241272f, -0.03045438416302204f, -0.4125255346298218f, 0.3683530390262604f, -0.2698453664779663f, -0.9857985377311707f, -0.47199076414108276f, -0.0783226415514946f, 0.5892837643623352f, 1.2498642206192017f, -1.234653115272522f, -0.0013108636485412717f, 0.7755194902420044f, 0.19315829873085022f, 1.3433040380477905f, -0.39465200901031494f, -0.5150922536849976f, 0.3830311596393585f, -0.9769555926322937f, 0.7637977004051208f, 0.12306158244609833f, -1.2370758056640625f, -1.2894343137741089f, -0.9705761075019836f, 0.9187757968902588f, 0.9821351766586304f, -0.4879138767719269f, -0.30360376834869385f, 2.0265355110168457f, 0.40967342257499695f, 0.7132695913314819f, 1.8868111371994019f, 0.6902899742126465f, -0.04476971551775932f, -0.03363388776779175f, 1.1792222261428833f, 0.8757511377334595f, -2.301875114440918f, 1.0762810707092285f, 1.4723693132400513f, 0.6662830114364624f, 0.22690092027187347f, -0.9138230085372925f, -1.457029104232788f, 0.03581761196255684f, -0.9291303157806396f, 0.4038029611110687f, -0.21844321489334106f, -0.4939993619918823f, 1.179942011833191f, 1.8594977855682373f, -0.4137862026691437f, 0.15132997930049896f, -0.0010087001137435436f, -1.0667427778244019f, 1.4108561277389526f, 0.24650610983371735f, 0.4927920699119568f, -0.6581372618675232f, -0.10233523696660995f, 0.12058872729539871f, 0.2761908769607544f, -0.7139783501625061f, 0.16629981994628906f, 1.1366126537322998f, -0.33142781257629395f, 2.6221468448638916f, 0.5184088349342346f, 0.739393413066864f, -0.8494256734848022f, -1.9893426895141602f, -1.3143675327301025f, 0.1328691989183426f, -0.5128253102302551f, 0.07802387326955795f, -0.5991530418395996f, -0.8056637048721313f, -0.5388752818107605f, 0.8337291479110718f, 2.1441681385040283f, -0.46184006333351135f, 0.6495042443275452f, 0.36905255913734436f, -0.7015367746353149f, 0.27310314774513245f, -1.6593921184539795f, 0.29731303453445435f, -1.31621515750885f, 0.12191462516784668f, 0.3006223440170288f, 0.5258087515830994f, 0.4770564138889313f, -0.20757682621479034f, 0.4625823199748993f, -0.9585452079772949f, 1.7045671939849854f, 1.419979453086853f, 0.09495851397514343f, 0.3802972435951233f, -0.4323408901691437f, 2.7312309741973877f, 0.3062905967235565f, 0.4344584047794342f, -1.2524477243423462f, 0.47808775305747986f, 0.07734657824039459f, -0.48799172043800354f, 0.7261772155761719f, -0.9319478273391724f, -0.8718891143798828f, 0.45363426208496094f, 0.3459012508392334f, 0.848518967628479f, -0.7525874376296997f, 0.19898031651973724f, 0.1414770483970642f, 0.43200522661209106f, 1.182044506072998f, 0.0033933203667402267f, -1.5844100713729858f, 0.2761341631412506f, 0.6477669477462769f, 1.1913690567016602f, 0.09115186333656311f, -1.5909662246704102f, -0.027118360623717308f, 1.2460919618606567f, 0.6424751281738281f, -0.047891248017549515f, -0.647229790687561f, -0.23609064519405365f, 0.07110362499952316f, -0.30918216705322266f, 0.32602694630622864f, -1.0240145921707153f, -2.5446670055389404f, -0.5895728468894958f, -0.4324244260787964f, -0.8140076398849487f, -0.3890652060508728f, -1.13597571849823f, -0.3532457649707794f, -2.3065085411071777f, -0.20395424962043762f, 0.6685602068901062f, -1.2822614908218384f, 0.30340856313705444f, 0.5633530616760254f, -0.09658124297857285f, -0.3894469738006592f, 0.22404761612415314f, 1.3719074726104736f, 0.569182813167572f, 1.1740480661392212f, -0.7359927892684937f, 0.5279164910316467f, -0.52259761095047f, 1.4638569355010986f, -1.2868918180465698f, 0.6853673458099365f, 1.0429801940917969f, 1.96529221534729f, -0.45007675886154175f, -0.5786357522010803f, 1.541925072669983f, 2.8543806076049805f, -0.7554785013198853f, -0.5379461050033569f, -0.7911050319671631f, 0.7011352777481079f, -1.4032478332519531f, -0.012685478664934635f, -0.5187733173370361f, 1.255434274673462f, 0.17988650500774384f, -0.13968797028064728f, 0.6899017095565796f, -0.9638485312461853f, 0.47390419244766235f, -1.083811640739441f, -0.108744777739048f, 0.8243650794029236f, 1.3675810098648071f, 0.7378400564193726f, -0.19896702468395233f, 0.6674330234527588f, 0.03600366786122322f, 0.24083632230758667f, -1.5012763738632202f, -0.7149558067321777f, -2.126762628555298f, -1.1807502508163452f, -1.3129348754882812f, -2.5667941570281982f, 0.6503364443778992f, -0.3889259994029999f, -0.4189043343067169f, 0.7983470559120178f, -0.3197358250617981f, -0.8505349159240723f, -1.3615714311599731f, -0.8035953640937805f, -0.06347727030515671f, 0.13253536820411682f, -1.9266542196273804f, 0.8539193272590637f, -0.13408313691616058f, -1.282929539680481f, 0.03780364990234375f, 0.7096128463745117f, 1.1662380695343018f, 0.8126105666160583f, 1.4384385347366333f, 1.8890222311019897f, -0.9130924344062805f, 0.03610055148601532f, -0.5193602442741394f, -1.3776048421859741f, 0.6756148934364319f, 0.7642406225204468f, 0.1278512328863144f, 0.512991189956665f, -1.0407685041427612f, 0.448485791683197f, -1.1702114343643188f, 0.8198426961898804f, 0.01693599671125412f, 1.4981187582015991f, -0.7068426609039307f, 0.5934628248214722f, 1.919209361076355f, 1.3406710624694824f, 0.07648162543773651f, -0.4410470724105835f, -0.0978068932890892f, 1.095009684562683f, 1.0229133367538452f, 0.5397310256958008f, -0.7647228837013245f, -0.590737521648407f, -0.10318559408187866f, 0.6214459538459778f, 0.5325868129730225f, 0.043896086513996124f, -1.2519514560699463f, 0.0696544423699379f, -1.6514869928359985f, 0.9199973940849304f, 0.34005025029182434f, 0.1417587399482727f, 1.8445940017700195f, 0.3398909568786621f, -0.5557951331138611f, 0.5655050277709961f, -0.05528254434466362f, 0.8540631532669067f, 1.1894739866256714f, 0.42318588495254517f, -0.6035352349281311f, 1.4443233013153076f, 3.0250484943389893f, -1.6034338474273682f, 2.1477253437042236f, -0.3787555396556854f, 1.4557304382324219f, 1.1084681749343872f, -1.184537410736084f, 0.7199674844741821f, 0.7042727470397949f, 0.5057917237281799f, 1.204850673675537f, -0.4900680184364319f, 0.7606944441795349f, -0.33889833092689514f, -0.1742597371339798f, 0.23202891647815704f, 1.3462589979171753f, -0.42982181906700134f, -0.6604134440422058f, -1.5597758293151855f, -0.3461014926433563f, 0.5544233322143555f, 1.3835493326187134f, 0.41140761971473694f, 0.7098760008811951f, 0.22245365381240845f, -0.982473611831665f, -0.35945725440979004f, -0.7463049292564392f, 0.5179733633995056f, 0.6092063784599304f, 0.5064983367919922f, 0.8556069731712341f, 0.5761587619781494f, 0.11352583765983582f, -0.8009540438652039f, -0.2633814215660095f, 1.5817502737045288f, -1.2247875928878784f, 0.23484112322330475f, -1.407819390296936f, -0.09603477269411087f, -0.294853538274765f, -0.5165784358978271f, 0.2704370319843292f, -1.0562596321105957f, -0.8496796488761902f, -0.2919580638408661f, 0.10698884725570679f, -0.9970720410346985f, 0.05950453504920006f, 0.6107109189033508f, 0.5484238862991333f, 0.7350953221321106f, 0.962885856628418f, 0.08861476927995682f, -0.0801108106970787f, 1.0644340515136719f, -0.2798626720905304f, -0.6881742477416992f, -1.1818499565124512f, 0.24130821228027344f, -0.6987038254737854f, 0.45643651485443115f, -0.10149910300970078f, -0.34807199239730835f, 1.7285562753677368f, 0.2142704576253891f, 0.45765289664268494f, -0.269319087266922f, -1.5785412788391113f, -0.3476867079734802f, 0.5194124579429626f, 1.4888246059417725f, 1.0837124586105347f, 0.4755038022994995f, -1.0340323448181152f, 0.18275369703769684f, -0.20516234636306763f, -0.3147018849849701f, 0.08301469683647156f, 0.2176828533411026f, 0.2920781075954437f, -0.17470814287662506f, 0.9676891565322876f, 0.45085909962654114f, 0.6715953946113586f, 0.8490674495697021f, 1.170888900756836f, 0.8825610876083374f, 0.17298388481140137f, -1.4316335916519165f, -0.4947497546672821f, 0.6246524453163147f, -0.7811664342880249f, -0.41328999400138855f, 0.712009072303772f, 1.1277714967727661f, -0.6925867199897766f, -1.7561272382736206f, -0.6673583388328552f, 0.04091703146696091f, -0.060151856392621994f, 0.2014705091714859f, 2.1779797077178955f, -0.23840203881263733f, 0.5123522281646729f, 0.1427735835313797f, -0.8951197266578674f, -0.7939775586128235f, 0.6872723698616028f, 0.3946205973625183f, -0.9005926847457886f, -1.5005147457122803f, -0.8442767858505249f, 1.4259073734283447f, 1.6183183193206787f, -0.06933283060789108f, 0.06978437304496765f, 0.3839779198169708f, 1.7791979312896729f, 0.5468734502792358f, -0.9818529486656189f, 0.6328914165496826f, 0.5340191125869751f, -0.674835205078125f, 0.7835897207260132f, 1.1304746866226196f, 0.8906894326210022f, -0.2404831200838089f, -0.32920387387275696f, 0.5127183794975281f, -0.26440465450286865f, 0.2585761547088623f, -1.6634936332702637f, 1.2309634685516357f, 0.25832492113113403f, -0.06058019772171974f, 1.1258721351623535f, -1.048923373222351f, -0.88661128282547f, -0.3876877427101135f, -1.1108732223510742f, 0.8258382678031921f, 0.47655177116394043f, -0.4854878783226013f, -0.1140289157629013f, -0.4026731848716736f, 1.354137897491455f, 0.4775455594062805f, -0.7650561332702637f, 1.2286686897277832f, -2.4340736865997314f, -0.5304896831512451f, 0.25538599491119385f, -0.5224623084068298f, 0.7963698506355286f, 0.44965043663978577f, -3.106328010559082f, 0.9458276629447937f, -0.8396266102790833f, -0.06616523861885071f, -0.8452204465866089f, 1.9770677089691162f, -0.07092823088169098f, -1.4889180660247803f, 1.2306435108184814f, 0.7042104005813599f, -0.3497500419616699f, -2.0363707542419434f, -0.4588965177536011f, -1.1338030099868774f, 0.6588250994682312f, 0.3726101517677307f, -0.9897713661193848f, -0.15447334945201874f, 0.33320167660713196f, -0.9029294848442078f, 0.3004419207572937f, 0.0267170500010252f, -0.3697128891944885f, -1.2930028438568115f, 0.4252126216888428f, -0.05628490820527077f, -1.338055968284607f, 0.5246880650520325f, -0.9283785820007324f, -0.14128278195858002f, -0.2966694235801697f, -1.9104946851730347f, -0.6022037863731384f, -1.6013476848602295f, -1.2525506019592285f, 0.6440190076828003f, 1.6395184993743896f, -0.25020739436149597f, -0.24597389996051788f, 0.8795658946037292f, -0.016383398324251175f, -1.4897207021713257f, -0.4389103353023529f, -0.6970252990722656f, -0.17175325751304626f, -0.6456266641616821f, 0.23320063948631287f, 0.26085028052330017f, -0.7153372168540955f, -0.059470854699611664f, -0.5745509266853333f, 0.7591805458068848f, -1.0744175910949707f, -0.09136287868022919f, 0.3293827772140503f, 0.07306429743766785f, -0.10749480873346329f, -1.5194628238677979f, -0.5850174427032471f, -0.08793152123689651f, -0.6866653561592102f, 0.4101375937461853f, -1.3473118543624878f, 1.4177610874176025f, -0.467404842376709f, -0.9928666353225708f, -1.9059150218963623f, -2.0202691555023193f, 0.32121846079826355f, -0.5283262729644775f, 0.6973733305931091f, 0.9967362284660339f, -1.3085604906082153f, 0.32580918073654175f, 1.8071491718292236f, -0.27431318163871765f, -0.1326882690191269f, 0.32671934366226196f, -0.8231000304222107f, 0.6738032102584839f, 0.551419734954834f, 1.1634254455566406f, -0.9665390253067017f, -0.6739510893821716f, 0.2892301678657532f, -0.46433013677597046f, 0.8342204093933105f, 1.0237910747528076f, 0.659813642501831f, -1.458429217338562f, -0.7326241135597229f, 1.2922906875610352f, 1.6295740604400635f, -0.8344282507896423f, -0.539470374584198f, 1.4665507078170776f, 2.654942035675049f, 1.609459400177002f, 0.367727130651474f, -0.9192121624946594f, 0.35196706652641296f, -0.15669254958629608f, -1.098692774772644f, -1.2132517099380493f, -0.07032525539398193f, 1.8989076614379883f, 0.4093967378139496f, -1.4458993673324585f, -1.5456795692443848f, -0.4921732246875763f, 0.09215065091848373f, -1.1077474355697632f, 1.0582451820373535f, -0.29416555166244507f, -0.5785751342773438f, 1.212047815322876f, 2.0987792015075684f, -1.5964131355285645f, 0.5674765110015869f, 0.9744538068771362f, 0.27448296546936035f, -0.04056643322110176f, -0.5840954780578613f, -0.31466683745384216f, -0.16959930956363678f, 0.658042311668396f, 0.8539144992828369f, -0.7443782091140747f, -0.1468161642551422f, 1.0819309949874878f, 1.2373405694961548f, -1.3463493585586548f, -0.5285679697990417f, -1.5322294235229492f, 0.8283826112747192f, 0.45322731137275696f, -0.3450125455856323f, -0.2933650016784668f, 0.10613418370485306f, 0.11260014772415161f, 0.02781728096306324f, 0.5361921191215515f, -1.0331605672836304f, 1.0188977718353271f, -0.8913255929946899f, 0.8866236805915833f, 0.8713390231132507f, -0.4831593930721283f, -1.7715193033218384f, 0.8562206029891968f, -3.832531690597534f, -0.9927570819854736f, -0.11961783468723297f, 1.397810459136963f, -0.3067088723182678f, -1.4679176807403564f, 0.0910743772983551f, 1.2350000143051147f, 0.5837976336479187f, -1.8317182064056396f, 0.1937909871339798f, -0.8611426949501038f, -0.522757887840271f, 1.7186238765716553f, 0.0968942642211914f, 0.4322175085544586f, 0.11584769189357758f, -0.2557562291622162f, 1.1861584186553955f, -0.9166569709777832f, 0.8642276525497437f, -1.716816782951355f, 0.6718529462814331f, -0.21214154362678528f, 1.0166347026824951f, 0.2806217670440674f, 1.9681813716888428f, -0.27264782786369324f, 1.2400078773498535f, -0.5684311389923096f, 0.36735019087791443f, 0.24113479256629944f, 1.9915446043014526f, -0.9585699439048767f, -1.2203160524368286f, -0.7793720364570618f, -1.0658658742904663f, -0.5502451658248901f, 0.9851812124252319f, 1.387345790863037f, -0.794191300868988f, 0.6909231543540955f, -0.7403607368469238f, 0.9804211258888245f, -0.905766487121582f, -2.9151322841644287f, 0.47318676114082336f, -0.05469474941492081f, 1.0238094329833984f, -0.5702749490737915f, 0.2909986078739166f, -0.41753849387168884f, -1.0129939317703247f, 0.535078763961792f, -0.7609738707542419f, -0.8248465657234192f, 0.36742645502090454f, 0.7183824777603149f, -0.8665743470191956f, -0.17533333599567413f, 0.768028736114502f, 1.0834310054779053f, 0.5876799821853638f, 0.04771281033754349f, 2.1327197551727295f, 0.19281719624996185f, -0.07964225858449936f, 1.1059718132019043f, -0.9939178228378296f, -1.3392163515090942f, -1.2726119756698608f, 0.3544987440109253f, 0.42930465936660767f, -0.5719326138496399f, -0.30639445781707764f, -0.12276917695999146f, 1.62221360206604f, 0.07731132954359055f, 0.183979332447052f, -0.8637551665306091f, 0.23347528278827667f, 0.3847457766532898f, 1.3200364112854004f, 0.2528532147407532f, 2.9082677364349365f, 1.2357676029205322f, -0.6267421841621399f, -0.28073710203170776f, 2.0288591384887695f, -0.46663370728492737f, -0.5359372496604919f, 0.7414069175720215f, 0.08158037811517715f, 1.2316521406173706f, -0.8425887823104858f, -1.141897201538086f, 1.2004878520965576f, -0.8595814108848572f, -1.5196866989135742f, -0.10754015296697617f, 1.4483332633972168f, -2.0370566844940186f, 1.3712586164474487f, -0.243259459733963f, -0.7683921456336975f, 0.1017654687166214f, -0.35750812292099f, 0.35395190119743347f, 0.20281589031219482f, 2.9194436073303223f, -1.6971954107284546f, -0.8292406797409058f, -1.2668757438659668f, 1.2814747095108032f, -0.29335519671440125f, 0.7705280780792236f, -0.5614521503448486f, 1.4171453714370728f, 0.23598231375217438f, -0.29366371035575867f, -1.2262547016143799f, 0.38033702969551086f, -1.2398927211761475f, -0.5345798134803772f, 0.33024173974990845f, 1.9377914667129517f, 1.0871516466140747f, 1.3149042129516602f, -0.9444983005523682f, 0.1276102066040039f, 2.106604814529419f, -1.1304327249526978f, -0.9464563131332397f, 0.1686755269765854f, -0.446566104888916f, -0.6286743879318237f, -0.11334973573684692f, -1.962886095046997f, -1.523536205291748f, 0.645367443561554f, -0.9533721804618835f, -0.5536230206489563f, 0.6801379919052124f, 1.2470786571502686f, -0.1793764978647232f, 0.6101059317588806f, -0.10875027626752853f, 0.996456503868103f, -0.7419731020927429f, -1.1421011686325073f, -1.1778020858764648f, -0.04226749762892723f, 0.30849045515060425f, -0.7805798053741455f, -0.8158796429634094f, -2.9890658855438232f, 1.5734566450119019f, -1.3029974699020386f, 1.1616994142532349f, -0.25818052887916565f, 0.31204938888549805f, -1.2942535877227783f, 0.608341634273529f, -1.1809622049331665f, 0.15562251210212708f, 0.6069639921188354f, 1.4125137329101562f, -0.27004849910736084f, -0.4410642385482788f, -0.1343953162431717f, 1.1372771263122559f, 0.18370741605758667f, 1.8697383403778076f, 1.0696011781692505f, -0.17657233774662018f, -1.340453863143921f, 0.717379093170166f, 0.6038413643836975f, 0.7894347310066223f, 0.9626035690307617f, -0.25843867659568787f, -0.8331825137138367f, -0.02316661737859249f, 1.4387904405593872f, -0.77969890832901f, -0.3609391152858734f, 0.21925963461399078f, -0.4726978540420532f, -1.0638948678970337f, -0.4561823606491089f, 0.5214431285858154f, -0.5561836957931519f, -0.2302778661251068f, 1.117246150970459f, 0.7824702858924866f, -1.1049346923828125f, -0.7501540780067444f, -0.47921040654182434f, -0.011093219742178917f, 0.03258634731173515f, -0.756567120552063f, 0.104627326130867f, 0.4133651852607727f, -0.958824634552002f, -0.2272576242685318f, 1.3634916543960571f, -2.3571479320526123f, 1.173104166984558f, 0.7147674560546875f, 0.4163925051689148f, -0.06465863436460495f, -0.7909473180770874f, 1.2354754209518433f, 0.2998451292514801f, -0.9952824711799622f, -0.5479734539985657f, 1.514818549156189f, -0.325872004032135f, 0.0061524491757154465f, -1.5123530626296997f, 0.25006136298179626f, -2.4219810962677f, -0.8351162672042847f, 0.6859942674636841f, 1.443666696548462f, 0.6747081875801086f, -0.0002302070497535169f, -0.21609316766262054f, 1.0141247510910034f, 0.721377432346344f, -0.29935362935066223f, -0.49368005990982056f, -1.5796416997909546f, 0.3189202845096588f, -0.5672794580459595f, 1.5067633390426636f, 1.161847710609436f, -0.08308011293411255f, -2.262038469314575f, 0.8626111745834351f, 0.196146160364151f, 0.2249874323606491f, 0.6830949187278748f, 0.0019485306693241f, 1.01323664188385f, -0.6184468865394592f, 0.7670295834541321f, 2.8818862438201904f, 0.6386357545852661f, -0.10977188497781754f, -0.17038141191005707f, -0.938051164150238f, -0.11422315984964371f, 0.1034913882613182f, -1.2966164350509644f, -0.41023650765419006f, -0.8332573175430298f, -0.9556332230567932f, 0.1063748151063919f, -0.20979070663452148f, 0.6346396803855896f, 0.545662522315979f, -0.937210738658905f, -1.1672022342681885f, -0.4361383616924286f, 0.0964970588684082f, -0.3027929663658142f, -0.638506293296814f, -0.05629456788301468f, 0.11661393195390701f, 0.3269132375717163f, -0.7588294148445129f, -0.952683687210083f, -0.7779799103736877f, 0.3503226041793823f, 1.2010222673416138f, 0.8768793344497681f, -0.7691330313682556f, -2.330476999282837f, 1.9413354396820068f, -1.0022987127304077f, -1.4932167530059814f, -1.2868070602416992f, 0.21946865320205688f, 0.8497498631477356f, -0.02522038109600544f, 0.6064606308937073f, 1.6981608867645264f, -0.36366915702819824f, 0.6935068964958191f, 0.12109924107789993f, 0.675596296787262f, 0.8142848610877991f, 0.0793362557888031f, -0.7808834314346313f, -1.163609266281128f, 0.3780289888381958f, 0.5237964987754822f, -1.3662828207015991f, -0.439242959022522f, -0.8599120378494263f, 0.3788713812828064f, -0.46068817377090454f, 0.7436892986297607f, -1.2479252815246582f, -0.4358613193035126f, 0.2984321117401123f, -1.8900177478790283f, 0.19737061858177185f, -0.7584667801856995f, 0.8250064849853516f, -1.5966553688049316f, 0.04276036471128464f, 0.7530690431594849f, -0.04625223949551582f, -0.1391070932149887f, -0.6105663776397705f, 0.24455690383911133f, -0.8800807595252991f, -0.03276701644062996f, -0.09261447191238403f, -0.9824548959732056f, 1.3447729349136353f, 0.19431965053081512f, -0.6367602348327637f, 0.941990315914154f, 1.2206652164459229f, 0.08320564776659012f, 0.5858650803565979f, -0.2219216674566269f, -0.6149584650993347f, -0.018692156299948692f, 1.0629346370697021f, -0.8915789723396301f, -1.4766337871551514f, 1.0600273609161377f, 0.6533640027046204f, -0.9062879085540771f, 1.4614392518997192f, 1.6020095348358154f, -0.8768263459205627f, 0.43399056792259216f, -0.06297583878040314f, -0.9222075343132019f, -1.3687634468078613f, 0.5819101333618164f, 1.2366944551467896f, 1.6560721397399902f, 1.2221823930740356f, 1.4343817234039307f, 0.982934296131134f, 0.003909424878656864f, 0.16369947791099548f, 1.25956130027771f, 1.0566132068634033f, -1.0371782779693604f, -1.5509816408157349f, 1.1234275102615356f, 1.1463638544082642f, -0.3711417317390442f, 1.1168509721755981f, -1.9369032382965088f, -0.8143561482429504f, 1.0661178827285767f, 0.771891176700592f, -1.9291707277297974f, 0.034095875918865204f, -0.6951659917831421f, 1.3456140756607056f, 0.34626761078834534f, 0.8155362606048584f, -0.7486876249313354f, -0.7881835699081421f, 0.5057575702667236f, 1.2215378284454346f, -0.9714295268058777f, 0.27692535519599915f, -1.5334482192993164f, 1.146209955215454f, -0.18189188838005066f, -1.2797164916992188f, -0.5713837146759033f, 1.1689343452453613f, -1.8829604387283325f, -0.369537353515625f, -2.0042972564697266f, -0.8240620493888855f, -0.3844030499458313f, 0.5684375166893005f, -1.137097716331482f, -0.31372663378715515f, 0.15282166004180908f, 1.0580254793167114f, -0.17965702712535858f, -1.178733229637146f, -1.2379671335220337f, -1.5433486700057983f, -0.6671661734580994f, 0.9025653600692749f, 0.304572194814682f, -0.47141557931900024f, 0.005512263625860214f, 0.8932762742042542f, 0.14350247383117676f, 0.7622402906417847f, -0.758182168006897f, -0.7234253287315369f, 0.7250988483428955f, -0.05147926136851311f, -0.6577823162078857f, -0.036672789603471756f, 0.514220118522644f, -0.6020243763923645f, -0.09203547239303589f, -1.7167327404022217f, -0.7002271413803101f, 0.4779217541217804f, 0.46033239364624023f, -0.386881560087204f, -0.0812682956457138f, 0.5568539500236511f, 0.04228341206908226f, -0.3627345860004425f, -1.3895257711410522f, 0.17128385603427887f, -1.2317392826080322f, 0.6718097925186157f, -0.15104348957538605f, 0.3213997781276703f, 0.9548746347427368f, 0.04617787152528763f, 1.7811017036437988f, 0.8509474396705627f, 1.179113507270813f, -0.35717684030532837f, 1.1261653900146484f, 1.2983627319335938f, -0.6900910139083862f, 0.4424906373023987f, 1.1874427795410156f, -0.21165163815021515f, -1.2463988065719604f, 0.9242297410964966f, 0.13773603737354279f, -0.060615699738264084f, 0.18482893705368042f, 0.09389957040548325f, -0.2936820983886719f, 0.36884430050849915f, -1.0105520486831665f, -1.1568186283111572f, 0.04061844199895859f, 1.7561308145523071f, -0.5621538162231445f, 0.19418247044086456f, 0.027558235451579094f, -0.1787608563899994f, -1.4996418952941895f, 0.26971569657325745f, 1.2250828742980957f, -1.170371651649475f, -0.11677189916372299f, -1.35635244846344f, 0.5242968797683716f, 1.0622302293777466f, -0.620258629322052f, -1.7659958600997925f, -0.06464217603206635f, 0.21129246056079865f, 0.825295627117157f, -0.4999869465827942f, 2.0010547637939453f, -0.8498311638832092f, -0.5403968691825867f, 0.628537118434906f, -0.7642602920532227f, -2.773613929748535f, -0.22911065816879272f, -1.0603324174880981f, 1.0186315774917603f, 1.763922929763794f, -1.4548780918121338f, -2.5379507541656494f, 3.445625066757202f, 1.4860185384750366f, 2.268347978591919f, -0.5500510334968567f, -0.12460697442293167f, 1.166976809501648f, 1.2409895658493042f, -0.7066188454627991f, 0.9183834791183472f, -0.02982438914477825f, -0.3448553681373596f, 1.0654057264328003f, -0.15129554271697998f, -0.048379626125097275f, -0.5897568464279175f, 0.09694309532642365f, -1.1129159927368164f, 0.5585124492645264f, -1.7732727527618408f, 0.023851748555898666f, -1.156516194343567f, -0.34847185015678406f, -1.6212294101715088f, -0.8558416366577148f, 0.40576910972595215f, -0.91661536693573f, -1.0765177011489868f, 0.5447612404823303f, 1.1705763339996338f, 0.9127081632614136f, -0.6960206627845764f, -0.7912065982818604f, -0.4341987073421478f, 0.3491472005844116f, -0.9907275438308716f, -1.5203826427459717f, 0.9009959697723389f, 0.07418181002140045f, -0.9035959839820862f, 0.9040606021881104f, 0.25104597210884094f, 0.4702746272087097f, -0.5477657318115234f, 1.522364616394043f, 1.6411151885986328f, -1.3906173706054688f, 0.5155009627342224f, 0.37119555473327637f, -0.015211731195449829f, 0.8483667373657227f, 0.634861409664154f, 0.5293999314308167f, -0.18842656910419464f, 1.757521390914917f, 1.3967915773391724f, -0.5659265518188477f, 0.1281542032957077f, 1.8777929544448853f, -0.3289257884025574f, 0.02351505309343338f, 0.428303986787796f, -0.7096118927001953f, 0.723761260509491f, 1.5117958784103394f, 0.5427215099334717f, 2.0354838371276855f, 1.0238486528396606f, -0.39082857966423035f, -1.2726235389709473f, -0.6254484057426453f, 0.9178156852722168f, 0.3840969204902649f, -0.19802889227867126f, 0.5223742127418518f, 0.05841278284788132f, 0.28558799624443054f, -1.0703928470611572f, 0.9307976365089417f, -0.18846826255321503f, -0.8914596438407898f, 0.12508316338062286f, 0.3772087097167969f, 0.9574744701385498f, -1.9291036128997803f, 0.5764585137367249f, 0.20935998857021332f, 0.5120382905006409f, -0.7815778851509094f, -1.478035569190979f, 0.051756951957941055f, 2.1100034713745117f, 0.020343216136097908f, -0.6159464716911316f, -0.4243218004703522f, -0.5821011662483215f, 0.5247467160224915f, -0.8761705756187439f, 0.48434850573539734f, 0.019129564985632896f, 0.03497670218348503f, -0.35594069957733154f, 1.505028486251831f, -0.8405776023864746f, -0.15094131231307983f, -0.5910253524780273f, 0.4260155260562897f, 0.7726138234138489f, 0.9289091229438782f, -1.0194525718688965f, -0.4821736812591553f, -0.4351550042629242f, 0.3517809212207794f, 1.2222594022750854f, -0.030398759990930557f, -1.0700304508209229f, -0.4833625555038452f, -0.5188538432121277f, -1.1346086263656616f, -1.0445209741592407f, 0.4199317395687103f, 0.8357481956481934f, 0.9475129246711731f, -0.34265488386154175f, -0.9238923788070679f, 0.3848164975643158f, 0.4773769974708557f, -1.2878144979476929f, 0.2491273283958435f, 0.32681646943092346f, 1.0925219058990479f, -0.7518913149833679f, -1.2260730266571045f, -0.3927396237850189f, -0.7111839056015015f, 0.5547724366188049f, 1.4059346914291382f, -0.22924941778182983f, 0.4364280700683594f, 1.2370060682296753f, 1.1473842859268188f, 0.29434895515441895f, -0.1789979487657547f, 1.9290111064910889f, 1.1900452375411987f, -0.10487240552902222f, -0.5064011812210083f, 2.4400694370269775f, -0.33963847160339355f, 1.7511601448059082f, 1.3401062488555908f, -0.9492565989494324f, -0.8270853161811829f, -1.2403943538665771f, -0.2053070068359375f, -0.2497664988040924f, -0.7054344415664673f, -1.7715047597885132f, 0.4937354028224945f, -0.17678959667682648f, 1.410936951637268f, 2.476804733276367f, -0.8441710472106934f, -1.912861704826355f, 0.007326157763600349f, 0.19589348137378693f, 1.5108956098556519f, 1.0457415580749512f, 1.560778021812439f, 0.24919238686561584f, -1.4739270210266113f, 0.22397132217884064f, 1.1544770002365112f, 0.4349344074726105f, 0.5612083077430725f, 0.6398148536682129f, 0.7980097532272339f, 0.5769069790840149f, -0.22143854200839996f, 0.31076598167419434f, -0.052180398255586624f, 1.7757917642593384f, -1.198473572731018f, -1.1297553777694702f, -1.0952261686325073f, -1.1415528059005737f, 0.5066304802894592f, -0.6875526905059814f, -1.7462857961654663f, 0.06312361359596252f, -1.7986558675765991f, 0.6912683248519897f, 0.49413225054740906f, 0.14730526506900787f, 2.274590015411377f, -1.4762635231018066f, 1.167490005493164f, -0.18522614240646362f, 0.37665531039237976f, -2.800556182861328f, 1.1855055093765259f, 0.7821402549743652f, 0.2779245674610138f, -0.489844411611557f, 0.7103408575057983f, -0.6894504427909851f, -0.6078459620475769f, 0.5198960900306702f, -0.18495284020900726f, -1.3136197328567505f, -0.783242404460907f, -0.47829392552375793f, 1.730208158493042f, 1.0595130920410156f, -0.251127690076828f, 1.2796905040740967f, 1.1880545616149902f, 0.010817415080964565f, 1.351500153541565f, 0.399687796831131f, -0.10175959765911102f, -0.2829807698726654f, -0.5876336097717285f, -0.7546091079711914f, -1.0380902290344238f, -0.6061143279075623f, -0.26778313517570496f, -0.11727923899888992f, 2.056168794631958f, 1.2978315353393555f, -0.527733564376831f, 0.2199985533952713f, 2.055619716644287f, 0.3816293179988861f, -0.8949641585350037f, 0.6981958150863647f, -0.9663392901420593f, 0.6600029468536377f, -0.3451770842075348f, -0.033411234617233276f, -0.10130416601896286f, 0.6449755430221558f, 1.5684525966644287f, -0.630508303642273f, -0.23472319543361664f, 0.3285580277442932f, -0.47698524594306946f, 0.3249095380306244f, 0.6602651476860046f, -1.652677059173584f, -1.596110463142395f, 0.05211031809449196f, -1.4231536388397217f, -0.16193436086177826f, -0.7287773489952087f, -0.8276495337486267f, -0.9271824359893799f, -0.24771355092525482f, -0.283514142036438f, -1.265464186668396f, -1.345624327659607f, 0.14869070053100586f, -0.5625298023223877f, 1.319007396697998f, -1.1078379154205322f, -0.38139888644218445f, 0.673722505569458f, 0.2882064878940582f, -0.7846477627754211f, 0.886084258556366f, -1.4957531690597534f, -0.3524166941642761f, -0.6002296209335327f, -0.5345580577850342f, -1.4335700273513794f, -0.06124546006321907f, -1.0124340057373047f, 0.7134581804275513f, -0.3809734880924225f, 1.6037938594818115f, -0.6744462847709656f, 0.03632983937859535f, 0.2812545895576477f, 2.2968475818634033f, 1.8312735557556152f, 0.0294801015406847f, 0.7718657851219177f, -0.6380594372749329f, -0.05797451362013817f, -1.4051520824432373f, -0.8756160736083984f, -0.9422666430473328f, -0.14667245745658875f, -1.9506268501281738f, -1.632828712463379f, 0.7152358293533325f, -1.8892930746078491f, 2.4672696590423584f, 0.08700139075517654f, 1.649484395980835f, -0.6159215569496155f, -0.5199353098869324f, 0.01673818565905094f, 0.5336771607398987f, 0.2974875271320343f, 1.8911962509155273f, 0.009384111501276493f, 1.4318883419036865f, -0.49658793210983276f, -0.6031076312065125f, -0.32575365900993347f, 1.9036880731582642f, -1.842443585395813f, -0.16547387838363647f, -0.25706610083580017f, 1.3179066181182861f, -0.6072822213172913f, -0.16537998616695404f, 0.6803566813468933f, 0.1680675745010376f, 1.6327744722366333f, 1.3595441579818726f, 0.6304294466972351f, 0.13153165578842163f, 0.36382463574409485f, -0.7942026853561401f, 2.110002040863037f, -1.1197839975357056f, 0.13227719068527222f, -0.3069077730178833f, 2.2180261611938477f, 0.7556464672088623f, -2.05973744392395f, -0.27732527256011963f, -0.12983323633670807f, -1.0805774927139282f, -1.4954025745391846f, -0.8487699627876282f, 0.8632985949516296f, -0.18222470581531525f, 0.3396982252597809f, 0.023615580052137375f, -0.5582384467124939f, -2.211786985397339f, -0.792870283126831f, 1.4188532829284668f, 1.240154504776001f, 1.2471561431884766f, 1.5289140939712524f, -0.24465647339820862f, 0.09731336683034897f, 0.8652670383453369f, -0.29969948530197144f, -1.841177225112915f, 1.1419644355773926f, 0.3660371005535126f, 1.273138165473938f, -0.7165514230728149f, 0.3878994286060333f, -0.5754631757736206f, 1.2297093868255615f, -0.4566229581832886f, -0.6573424935340881f, 0.7881364822387695f, 0.3378683924674988f, -1.988023042678833f, 0.7956935167312622f, 0.9823195338249207f, -0.4144572615623474f, 1.3572540283203125f, 0.3109254240989685f, -0.6753576993942261f, -1.5371414422988892f, 0.9313072562217712f, -2.2768142223358154f, 0.4319401681423187f, 0.07773400843143463f, -1.5976078510284424f, 1.848445177078247f, 1.5493229627609253f, 0.1915358453989029f, -1.266395926475525f, -2.1607372760772705f, 0.7240417003631592f, -0.7832543253898621f, 1.1483440399169922f, -0.10642467439174652f, -0.32092228531837463f, 1.0402002334594727f, 0.11275313794612885f, 0.37306782603263855f, 0.8337973356246948f, 1.8036106824874878f, 0.7735506892204285f, -1.1966158151626587f, -0.6088662147521973f, 0.16352267563343048f, -0.3072216808795929f, -0.5694004893302917f, 0.13301636278629303f, -1.508280873298645f, -0.8582409024238586f, -0.7953895330429077f, -1.2091906070709229f, -0.7641999125480652f, 0.2154434323310852f, 0.2061154842376709f, 0.3153077960014343f, -0.3388381898403168f, -0.6360014081001282f, -0.45390239357948303f, -2.7026309967041016f, 0.6710167527198792f, 0.8398252725601196f, -2.002260208129883f, -0.6277503967285156f, -1.3664311170578003f, -0.47482022643089294f, -1.1213213205337524f, 0.07767737656831741f, -0.44291654229164124f, -1.2841796875f, -1.0159707069396973f, 1.3361929655075073f, -0.46696820855140686f, -0.2509534955024719f, 1.4244478940963745f, -0.6108700037002563f, -0.4096096158027649f, -0.4688802659511566f, -1.2304245233535767f, -0.14591790735721588f, 0.9639950394630432f, -0.9581076502799988f, 0.8579769134521484f, 0.17619068920612335f, -0.009090420790016651f, 2.2291040420532227f, -0.599851667881012f, 0.5864706635475159f, -0.40187644958496094f, 0.7005379796028137f, 2.269195318222046f, 1.1897660493850708f, -0.530225396156311f, 0.22658583521842957f, 0.8770406246185303f, -0.42359691858291626f, 0.04045804589986801f, -0.3348093032836914f, -1.6933249235153198f, 0.578863799571991f, 0.253021240234375f, 0.8249357342720032f, 0.17797666788101196f, 0.21032211184501648f, -1.3109723329544067f, 1.4387873411178589f, 1.3104835748672485f, -0.4802001118659973f, 0.2532861530780792f, 0.34186819195747375f, -2.0921499729156494f, -1.4881683588027954f, 0.24693161249160767f, -1.7462626695632935f, 0.28935855627059937f, 1.5511679649353027f, 0.8895691633224487f, 0.4553866684436798f, -1.143309235572815f, -0.06552600860595703f, 0.03079218976199627f, -1.0684497356414795f, -0.3178943395614624f, 2.2536263465881348f, -0.19900000095367432f, 0.5933657884597778f, 1.5937228202819824f, 0.8582064509391785f, 0.11251302808523178f, 0.2912493050098419f, 0.12979160249233246f, 0.08444352447986603f, 2.015190601348877f, 0.5599141120910645f, 1.3394620418548584f, 0.6110420823097229f, -0.5921895503997803f, -0.16634242236614227f, -0.3773656487464905f, 1.1717573404312134f, 0.6101416945457458f, 1.917343020439148f, 2.563725471496582f, 3.035121202468872f, -0.534939169883728f, -0.4571112394332886f, 0.802577018737793f, -0.28734639286994934f, 0.12288182973861694f, 0.21209175884723663f, -0.996752142906189f, 0.6648997068405151f, -1.1771180629730225f, 0.517611563205719f, 2.260444164276123f, 0.8793008327484131f, -1.4391361474990845f, -0.47865501046180725f, -0.12678508460521698f, -1.1487538814544678f, 0.9171100854873657f, 0.9602430462837219f, 0.20192822813987732f, -0.6621134877204895f, 0.283156156539917f, 1.1861251592636108f, -0.6021904945373535f, -0.2499617040157318f, 1.7409440279006958f, -0.7325262427330017f, -0.3309515118598938f, -0.7796709537506104f, 1.6620672941207886f, -0.0578240342438221f, 0.2314254492521286f, 0.22709836065769196f, 2.065865993499756f, 0.4171837866306305f, 0.3846955895423889f, -0.01396828331053257f, 0.5790372490882874f, 0.2881997525691986f, 0.8778835535049438f, 0.5851041674613953f, -0.29608187079429626f, 0.33585336804389954f, -0.7194162011146545f, -0.07809803634881973f, 0.35557520389556885f, -1.7239394187927246f, 0.7924410104751587f, 0.030582552775740623f, 1.0178154706954956f, 1.6657739877700806f, 0.17066167294979095f, -0.2777872681617737f, -0.19989891350269318f, 1.6998308897018433f, 0.7318398952484131f, 1.2774115800857544f, -0.3473694622516632f, -0.7603669166564941f, 1.2199066877365112f, -0.3723330795764923f, -1.8119957447052002f, -0.9909027814865112f, -0.3076462149620056f, 0.01513738464564085f, -1.9378867149353027f, -1.8149548768997192f, 1.3613444566726685f, -0.7856798768043518f, -2.4690840244293213f, 2.432499408721924f, 0.27381452918052673f, 0.8801257610321045f, -0.49671268463134766f, 0.056602153927087784f, 1.435631513595581f, 0.36377331614494324f, 0.46456536650657654f, 1.9551998376846313f, 0.6760215759277344f, 1.1773208379745483f, -0.5477144718170166f, -0.48883718252182007f, -0.2574329078197479f, 0.46062397956848145f, -1.6101100444793701f, 0.2585121691226959f, 1.7188366651535034f, 0.5200220942497253f, -1.301006555557251f, -1.5038570165634155f, -0.31398269534111023f, 1.2563191652297974f, -0.5480050444602966f, -0.06532658636569977f, 2.6805763244628906f, -0.9649611115455627f, -0.18265697360038757f, 0.3190588653087616f, 0.006673657335340977f, 0.566218376159668f, 1.7436981201171875f, 1.3330131769180298f, 1.3450311422348022f, -1.0255969762802124f, 1.3098556995391846f, -0.4485261142253876f, 0.8979019522666931f, -0.1220834031701088f, -1.0596239566802979f, 0.14630116522312164f, -0.870776891708374f, -0.24674776196479797f, -1.2373387813568115f, 0.8207234740257263f, -0.5435441136360168f, 1.1541569232940674f, -0.38771313428878784f, 0.12649448215961456f, -1.2653104066848755f, 1.7627143859863281f, -0.2666304111480713f, 0.5257315635681152f, 0.6358874440193176f, 0.10120877623558044f, 0.17400647699832916f, 1.1357200145721436f, 0.036105670034885406f, -0.6797627210617065f, -0.16857317090034485f, 0.26855215430259705f, -1.4135205745697021f, -2.3971786499023438f, 0.7723278403282166f, -0.15770527720451355f, 0.45117413997650146f, -2.815762519836426f, 0.19697873294353485f, 0.2619018256664276f, -0.8475749492645264f, 0.47121289372444153f, 0.3821605145931244f, -0.2688539922237396f, 1.0989755392074585f, -1.0098387002944946f, 0.688886284828186f, 0.5349103808403015f, 2.787760019302368f, 0.38038408756256104f, 0.22671592235565186f, 0.5783246755599976f, 0.6597211360931396f, -0.28375279903411865f, -0.6992143392562866f, 0.7167068123817444f, -0.09313177317380905f, 0.6840168237686157f, -0.19578158855438232f, -0.9126741886138916f, -0.2800213098526001f, -0.3882693350315094f, 0.10380307585000992f, -1.9663856029510498f, -1.013744592666626f, -1.0242971181869507f, 0.8466978073120117f, 1.2233283519744873f, -0.6524525284767151f, -0.6867133378982544f, 1.1395643949508667f, -0.696540355682373f, -0.39359593391418457f, 0.5099198222160339f, -0.15132363140583038f, 0.6866339445114136f, 0.5310931205749512f, -1.3795819282531738f, -0.6988955140113831f, 0.3475550413131714f, 1.001903772354126f, 0.7994512319564819f, 0.8503185510635376f, 0.791765034198761f, -0.8879274725914001f, 1.02428138256073f, 0.1911749392747879f, 0.8436497449874878f, -0.02484050951898098f, -0.7801733613014221f, 0.625577986240387f, 1.5644149780273438f, 0.5320438146591187f, 1.0700255632400513f, -0.5013832449913025f, -0.3839770257472992f, 0.8672234416007996f, 0.8700053095817566f, 1.8044092655181885f, 0.5074418187141418f, -1.0803741216659546f, 1.7308460474014282f, -0.009462441317737103f, 1.9249420166015625f, -0.3632628321647644f, 0.6629142165184021f, -0.6219039559364319f, 1.013214111328125f, -1.5853021144866943f, -0.9035424590110779f, -0.1714278757572174f, 0.9996393918991089f, 0.2449631243944168f, 0.07007400691509247f, 1.4646685123443604f, 0.4391697347164154f, 1.424518346786499f, -0.32468941807746887f, 0.35460788011550903f, -0.3405316174030304f, -0.6941221952438354f, 0.6556671857833862f, -1.087321400642395f, -1.1486494541168213f, 2.4220407009124756f, 0.7684311866760254f, -1.5410068035125732f, -0.411981463432312f, 1.3056882619857788f, -0.338508278131485f, 0.2913444936275482f, -0.8357943892478943f, -1.1213253736495972f, 0.3143851161003113f, -0.42382940649986267f, -0.43293496966362f, -0.9816297292709351f, 0.058465536683797836f, -1.3252078294754028f, -0.7915613055229187f, 0.47723037004470825f, 0.43918681144714355f, -0.3302071988582611f, 1.7920981645584106f, -1.0610337257385254f, -0.09547755867242813f, -1.1624996662139893f, -0.6900646686553955f, 0.3918372392654419f, -0.0669134333729744f, 1.0711755752563477f, 1.3083903789520264f, -0.05556102097034454f, 0.7882350087165833f, 0.37722504138946533f, -0.3213580846786499f, 0.5956750512123108f, -0.5032515525817871f, -0.7394251227378845f, -0.7672699093818665f, -0.040719106793403625f, 0.15840405225753784f, -0.4783642590045929f, 0.4646506905555725f, -0.49036842584609985f, -0.6546841263771057f, 2.7124791145324707f, 0.4293099343776703f, -1.0469098091125488f, -1.0857888460159302f, -0.05841507390141487f, 0.5456286668777466f, 0.2792662978172302f, 2.1168277263641357f, -1.5103271007537842f, -1.307977557182312f, -1.9681334495544434f, -0.8398545384407043f, 1.3870203495025635f, 0.9826740026473999f, -0.2890629470348358f, 1.0335131883621216f, -0.19352838397026062f, 0.07122802734375f, -0.16152557730674744f, 1.0519514083862305f, -1.4766151905059814f, -1.2671267986297607f, 0.2393292933702469f, 1.2190254926681519f, 0.17837846279144287f, -1.3159326314926147f, 0.646186888217926f, 1.6116095781326294f, -0.05114508792757988f, 0.41359904408454895f, -0.20463405549526215f, -2.1418421268463135f, 1.7502615451812744f, -1.4018466472625732f, -0.5717736482620239f, -0.38931718468666077f, -0.9860185384750366f, 0.5779818296432495f, 0.4738202393054962f, -0.7853313088417053f, -0.1793491244316101f, -0.1285407692193985f, 0.9474418759346008f, -0.07980115711688995f, -0.6013848185539246f, -0.5239112377166748f, -0.1949448138475418f, 0.7724379897117615f, -0.11171314865350723f, 0.5611289739608765f, -1.5560880899429321f, 1.4754453897476196f, 1.4865750074386597f, -0.02104124240577221f, 0.031055578961968422f, 1.0900859832763672f, -0.5583686828613281f, 1.3067461252212524f, -0.9643430113792419f, 1.6326349973678589f, -1.144444465637207f, 0.13422854244709015f, 1.668276309967041f, -0.6358500719070435f, -0.8220173120498657f, 1.151316523551941f, -0.9632699489593506f, -0.17086558043956757f, 0.14713206887245178f, -0.13801060616970062f, -0.1489359736442566f, -0.06645472347736359f, -0.880412757396698f, 0.1913401335477829f, -0.05544678866863251f, -0.2204407900571823f, -1.1233654022216797f, -1.1245696544647217f, 0.674127995967865f, 0.2520127594470978f, 0.7975156307220459f, 0.698857843875885f, -0.4366423487663269f, -2.0883853435516357f, -1.366032361984253f, 0.09937097132205963f, -0.3652037978172302f, 1.2572640180587769f, -0.8791263699531555f, 1.3767470121383667f, 0.6487414836883545f, -0.22251465916633606f, -0.7945832014083862f, 1.3094075918197632f, 1.1708104610443115f, 0.9832903742790222f, -0.7685240507125854f, -0.5897563099861145f, -0.008548072539269924f, 0.7963455319404602f, -0.6708536148071289f, -0.16284485161304474f, -1.8156386613845825f, 0.1582425981760025f, 1.5490630865097046f, 0.6892044544219971f, -1.5244767665863037f, 0.1314752995967865f, -0.5694751739501953f, -0.07285541296005249f, 0.02161409705877304f, 0.27252840995788574f, 1.5375657081604004f, -0.16460900008678436f, 0.04459971562027931f, 0.4962165355682373f, 0.9521092176437378f, 0.18977606296539307f, 1.1129406690597534f, -1.742975115776062f, 0.015372429974377155f, 0.43921586871147156f, -0.4629494845867157f, 0.683012068271637f, 0.0549420528113842f, 1.1227099895477295f, -0.9156265258789062f, 0.7202309370040894f, -1.7771120071411133f, -0.49309614300727844f, -0.3537367880344391f, 0.6029451489448547f, 1.4749457836151123f, -1.2571945190429688f, 1.1716068983078003f, -1.2938947677612305f, -0.027610722929239273f, -0.04813985526561737f, -1.6245776414871216f, -0.1380276381969452f, -0.7646363377571106f, -0.24534782767295837f, 0.21846671402454376f, 0.8769089579582214f, -1.0646121501922607f, 0.50408536195755f, 0.15752233564853668f, -0.5226418375968933f, -1.4756296873092651f, 0.25705486536026f, -1.7179490327835083f, 1.3074569702148438f, -0.4480331838130951f, 1.1221709251403809f, 1.4023377895355225f, 1.4818451404571533f, -1.4184188842773438f, 0.2722645699977875f, 1.2556324005126953f, -1.6670103073120117f, 1.050782322883606f, 0.13771948218345642f, -1.1567072868347168f, 1.0360978841781616f, -0.8660301566123962f, -1.0625808238983154f, 1.0239890813827515f, 0.7085686922073364f, 1.8764188289642334f, 2.794304609298706f, 0.38867735862731934f, 1.6592570543289185f, -0.8973153829574585f, -0.428587943315506f, -0.8077448010444641f, -2.2226719856262207f, 1.3841030597686768f, 0.27507975697517395f, 1.8155642747879028f, 0.5318379998207092f, 1.278135895729065f, -0.632580041885376f, -1.036582350730896f, 0.294900506734848f, -0.8918796181678772f, -0.08464081585407257f, 0.7254555821418762f, 0.40998774766921997f, -0.0699654296040535f, 0.8239985108375549f, 2.0619850158691406f, -1.0847980976104736f, -1.5027097463607788f, 0.4682971239089966f, -2.0921216011047363f, -0.31479185819625854f, 0.35259687900543213f, -0.6293240189552307f, -1.997800350189209f, -0.6937687397003174f, 0.43931636214256287f, -0.2434539496898651f, -1.389511227607727f, -0.0009331775945611298f, 0.4944293797016144f, -1.3989880084991455f, 0.3048802316188812f, 0.612401008605957f, -1.0865437984466553f, -0.7029951810836792f, -0.651735246181488f, 0.02101830020546913f, -0.07500791549682617f, -1.676796793937683f, 1.5087897777557373f, -0.8013211488723755f, -1.030982494354248f, -1.4118940830230713f, -0.8552696704864502f, -0.1083751767873764f, 2.1871159076690674f, -0.6815629601478577f, -1.4610782861709595f, -0.5785004496574402f, 2.1495704650878906f, -0.179605171084404f, 1.1426492929458618f, -0.054547298699617386f, 0.40587425231933594f, 0.6724022030830383f, 0.19652637839317322f, -0.07756809890270233f, -1.4263975620269775f, -1.0503880977630615f, -1.5335310697555542f, 0.4794066250324249f, 1.1617707014083862f, 0.8538435697555542f, -0.20266015827655792f, -0.725740909576416f, -0.9262224435806274f, 0.8973854184150696f, -0.7538284063339233f, -0.8116031885147095f, 0.5351168513298035f, 1.988852858543396f, 1.0685486793518066f, -0.5014570951461792f, 0.7195857763290405f, 1.110618233680725f, 0.1288852095603943f, -1.2922874689102173f, 0.6346917748451233f, 1.5678290128707886f, 0.03146770969033241f, 1.8645329475402832f, -0.8617974519729614f, 0.051693860441446304f, -1.4663174152374268f, -0.26106584072113037f, -0.06878392398357391f, 0.8156912922859192f, 0.4850883483886719f, -2.2269856929779053f, 0.5959717631340027f, 0.21369199454784393f, 0.2891088128089905f, 0.3474847674369812f, -0.43252453207969666f, -0.5997704267501831f, 0.2910449504852295f, -0.8439115881919861f, -0.013285640627145767f, -0.5315346121788025f, 0.08023621141910553f, -0.6925831437110901f, -0.6154842972755432f, -1.4682725667953491f, 0.005962289869785309f, -0.17264695465564728f, -1.3461240530014038f, 0.5067698955535889f, -0.7377328872680664f, -0.006388451438397169f, -1.411285400390625f, 0.23984402418136597f, 0.4191838204860687f, 1.395574688911438f, 0.976121723651886f, 0.40693262219429016f, -0.6316997408866882f, 1.5522570610046387f, 0.2695809602737427f, 1.6630321741104126f, 1.0006709098815918f, -0.6625555157661438f, -0.05369861051440239f, 0.2828640341758728f, 0.8807763457298279f, -0.3764995038509369f, 1.8677223920822144f, 2.1043002605438232f, -0.8154667019844055f, -0.4016047418117523f, -0.07730520516633987f, 0.4082213044166565f, -0.7410086393356323f, -2.3087081909179688f, -0.031602293252944946f, -1.454459547996521f, 0.7048721313476562f, -0.5495360493659973f, -1.0822854042053223f, -1.1814165115356445f, -0.7662639021873474f, 2.26790452003479f, -0.9821573495864868f, -0.7700114250183105f, -0.6950612664222717f, -0.47602152824401855f, -2.168846368789673f, -0.4960733652114868f, 1.8063979148864746f, -2.195838451385498f, -1.2756946086883545f, -0.23146040737628937f, -0.6978404521942139f, 0.05868193879723549f, 0.3181943893432617f, -0.9001671671867371f, 1.4585676193237305f, 0.9991294741630554f, -0.958099365234375f, -1.5130436420440674f, 0.8987486362457275f, 0.6024074554443359f, 1.213675856590271f, -0.9290968179702759f, 0.9378061294555664f, 0.32025381922721863f, -0.6372569799423218f, 0.5549997091293335f, 0.4728609323501587f, 1.5382329225540161f, -2.230900526046753f, 2.1022186279296875f, 1.7723872661590576f, -0.29818224906921387f, 1.774613618850708f, -0.403571754693985f, -0.763476550579071f, -0.13895398378372192f, -1.8085858821868896f, -0.1992855966091156f, -0.38452786207199097f, 0.772721529006958f, -0.761553168296814f, 0.32450324296951294f, -0.6566532254219055f, 1.0444575548171997f, -1.0310343503952026f, 0.13313262164592743f, 0.32965973019599915f, -0.7564722895622253f, 1.5161906480789185f, -0.442477822303772f, -0.19663105905056f, -0.5199072957038879f, 0.19425715506076813f, 0.468250036239624f, 0.658501386642456f, -0.16666415333747864f, -0.4670298099517822f, 1.4936802387237549f, -0.8678151965141296f, -0.26301294565200806f, 0.20588766038417816f, 0.022425031289458275f, 0.6587340831756592f, -0.20476730167865753f, -1.0465848445892334f, -1.2654074430465698f, -0.3633078634738922f, -0.4297706186771393f, 0.6679531335830688f, 1.0863715410232544f, 0.7616772055625916f, -0.011849306523799896f, -1.2028323411941528f, 0.5853366255760193f, -0.10431576520204544f, 0.2191448211669922f, -0.8329556584358215f, -0.2799490988254547f, 0.6428387761116028f, 1.405016303062439f, 1.3759163618087769f, 1.4840714931488037f, -1.5530608892440796f, -0.9330264925956726f, -1.1588698625564575f, -0.48916497826576233f, -0.6450856328010559f, -0.11284743994474411f, -2.4587764739990234f, 0.7599589824676514f, 0.9755558371543884f, 0.05122251436114311f, 0.960921049118042f, -0.6847841143608093f, 0.40178006887435913f, -0.9434769749641418f, -0.23219886422157288f, -0.2043316662311554f, -0.8605437278747559f, -0.3255579173564911f, -0.7162472605705261f, -0.08609210699796677f, -3.6307897567749023f, -0.6838181614875793f, -0.489888072013855f, -1.0136138200759888f, -0.8829219937324524f, 1.1272135972976685f, -1.2231221199035645f, -0.7950332760810852f, 0.6623095273971558f, 2.028413772583008f, 0.8947616219520569f, -1.1885942220687866f, -0.400406152009964f, 0.9303890466690063f, -1.0271012783050537f, 0.60735023021698f, -2.186370372772217f, -1.2513858079910278f, -1.5937029123306274f, -1.391968846321106f, -0.7062578201293945f, 0.5445007681846619f, 0.0782814472913742f, -1.7333787679672241f, -0.8157885074615479f, 0.3931215703487396f, -0.11953603476285934f, 0.14065788686275482f, 1.6809849739074707f, -0.2840443551540375f, -1.4785282611846924f, 0.227834552526474f, 0.26440107822418213f, -0.0759536400437355f, 0.9481480121612549f, 0.8856539726257324f, -1.2799997329711914f, -0.2185828685760498f, -0.15562321245670319f, 1.1858826875686646f, 1.7509973049163818f, 0.4133126139640808f, 0.7854304909706116f, 0.64668208360672f, -0.11213088780641556f, 0.8463886380195618f, 0.045808374881744385f, -0.6186387538909912f, -0.5598823428153992f, 0.3789231479167938f, -0.4264816343784332f, 0.9161636829376221f, 0.13592901825904846f, 0.4121098220348358f, 0.40913498401641846f, 0.4465799629688263f, -0.07119281589984894f, 0.4638987183570862f, -0.5617773532867432f, -0.23070071637630463f, -0.5451535582542419f, 0.018565375357866287f, -0.10694064944982529f, 1.1308897733688354f, 1.4536970853805542f, 0.6201399564743042f, -1.4827039241790771f, 0.4850791096687317f, -0.28107115626335144f, -1.132474660873413f, 0.4989188015460968f, 1.1301037073135376f, -0.3190559148788452f, -0.02982708252966404f, -0.4788385033607483f, 0.3957928717136383f, 0.5253366231918335f, -1.6755516529083252f, 0.35308536887168884f, -0.12078773975372314f, -0.2962888479232788f, -0.08987566083669662f, -0.45044565200805664f, -1.0356383323669434f, 1.7253460884094238f, -2.3891191482543945f, -1.0028913021087646f, -0.3333589434623718f, 0.9801297783851624f, 1.2715272903442383f, -0.7523331046104431f, -0.13584411144256592f, 0.6945744156837463f, -1.9437434673309326f, 0.3301672339439392f, 1.6046744585037231f, -0.4702155888080597f, 1.209662675857544f, 0.8889638781547546f, 0.1620996594429016f, 0.12699493765830994f, 0.7177993655204773f, 0.06786083430051804f, 0.8935588598251343f, 0.11956664174795151f, 1.0848652124404907f, -0.47686854004859924f, 0.2491820603609085f, 0.730522096157074f, 0.09865526109933853f, -0.5309102535247803f, 0.08605138957500458f, -1.499106526374817f, 0.8767158389091492f, -1.1525547504425049f, -0.34564009308815f, -0.8810292482376099f, -1.5831094980239868f, -0.9634820222854614f, -1.0025994777679443f, 0.07062987983226776f, 0.11008060723543167f, 0.7735913395881653f, -0.0040825954638421535f, 0.8591998815536499f, -0.7495615482330322f, -0.38007304072380066f, -1.8807623386383057f, 0.10670913010835648f, 1.1232846975326538f, -0.15919449925422668f, -0.561027467250824f, 0.30757027864456177f, -1.5486440658569336f, -1.054305911064148f, -0.8691359758377075f, -0.06806328892707825f, -0.4993351697921753f, 0.1407848447561264f, 0.07298417389392853f, 1.068840742111206f, -1.1345824003219604f, -0.2619936764240265f, 1.4751683473587036f, 0.1493469625711441f, -0.7424628138542175f, 0.0515340156853199f, -0.4829641580581665f, -0.12765848636627197f, 0.048511698842048645f, -0.6109879612922668f, 1.034902811050415f, 1.2692794799804688f, 1.4105430841445923f, -0.2778305411338806f, -0.2591007351875305f, 1.035352349281311f, 0.4310223162174225f, -0.5226048827171326f, 1.6362823247909546f, 2.1157736778259277f, 0.22881479561328888f, 0.6379362940788269f, 0.4769587814807892f, 0.036591242998838425f, -0.7712311148643494f, 0.11032737791538239f, 1.141363263130188f, 2.2910208702087402f, 0.9097362756729126f, 0.015577469952404499f, 0.7960927486419678f, -1.0888819694519043f, -0.22314855456352234f, -0.42274436354637146f, -0.9645407795906067f, 0.26433172821998596f, -0.4537980556488037f, 1.0802478790283203f, 0.17047974467277527f, 0.5164321660995483f, 0.6420630812644958f, 0.12355764210224152f, -0.6113547086715698f, -0.07968119531869888f, 0.8360456228256226f, -0.9061261415481567f, 0.5791953802108765f, -0.9833644032478333f, 0.04046724736690521f, 0.0772174745798111f, 1.1407610177993774f, -1.1166253089904785f, 0.797391414642334f, -1.2114757299423218f, 0.845003604888916f, 0.917169451713562f, 0.6150916218757629f, -1.4388847351074219f, 0.5669911503791809f, -2.142704963684082f, -0.45466870069503784f, -0.6032977104187012f, -0.13450507819652557f, 0.18595486879348755f, 0.3553427457809448f, 1.6049801111221313f, 0.41566601395606995f, 1.1401103734970093f, 1.4051463603973389f, 0.6006536483764648f, 1.797692060470581f, -0.5229107737541199f, -0.20334042608737946f, -0.45935776829719543f, -0.15298272669315338f, 1.5941132307052612f, -0.1434730738401413f, 0.052168138325214386f, -1.3556065559387207f, 1.5627477169036865f, 1.9382938146591187f, 1.2685400247573853f, 2.026231527328491f, -1.1499669551849365f, -0.8773688077926636f, 0.07689109444618225f, 1.4310468435287476f, -0.18800176680088043f, -1.6620466709136963f, 0.719353199005127f, -0.291658490896225f, 2.5384137630462646f, 0.10889503359794617f, -0.5992977023124695f, -0.394126832485199f, -1.3852146863937378f, -1.254912257194519f, 0.1908181607723236f, -1.0126259326934814f, 0.722631573677063f, -0.34968942403793335f, -0.8847613334655762f, -0.984367311000824f, -0.9932076334953308f, 0.8806517720222473f, -0.09622633457183838f, -1.206941843032837f, -0.49620160460472107f, 0.6914839744567871f, -3.028942346572876f, 1.3682804107666016f, -0.4541301131248474f, 0.45961233973503113f, 0.36240431666374207f, 0.34856998920440674f, -0.821986973285675f, -2.4602458477020264f, 0.37095823884010315f, 0.0116643775254488f, -0.5293033719062805f, -1.6280213594436646f, -0.6806969046592712f, -0.1684180051088333f, 1.5591261386871338f, 0.43387478590011597f, 0.26209768652915955f, 1.2855982780456543f, 0.6979333758354187f, 1.5001028776168823f, -1.1095906496047974f, 0.581849217414856f, 1.3310784101486206f, 1.2475563287734985f, 1.4747735261917114f, 0.3026885390281677f, -1.3104248046875f, -0.5138301253318787f, 0.7339244484901428f, -1.0213230848312378f, -1.202392816543579f, 0.16470061242580414f, 2.572078227996826f, -1.3335460424423218f, 0.546902596950531f, 0.38991880416870117f, 0.15686734020709991f, -0.39347609877586365f, -0.30205461382865906f, 2.149627923965454f, -1.751050591468811f, -0.7567965984344482f, -1.4173346757888794f, 0.5380285382270813f, 0.0939386859536171f, 0.45474377274513245f, -0.6351379156112671f, 0.9299246072769165f, -0.4753718972206116f, 0.9432480931282043f, 0.5524624586105347f, 0.40374335646629333f, 0.6184999346733093f, 0.4535541534423828f, -1.0948821306228638f, -0.320241779088974f, -0.028895698487758636f, 0.7330871224403381f, -1.6074398756027222f, -1.227219581604004f, 1.0834808349609375f, 0.05628223717212677f, 0.20571336150169373f, 0.43112990260124207f, 0.3616485893726349f, -1.1379897594451904f, 0.2402925342321396f, -0.47906389832496643f, 1.4835309982299805f, -1.1785883903503418f, 0.5978415012359619f, 0.5928131341934204f, -0.6638352870941162f, -1.003272533416748f, -0.1671929657459259f, -1.1786675453186035f, 0.8089823126792908f, 0.1908469796180725f, 1.2084790468215942f, -2.24063777923584f, -0.5793269276618958f, -1.1207023859024048f, -1.1295276880264282f, -0.5763978362083435f, -0.26363199949264526f, 0.789819061756134f, -0.4906677007675171f, 0.371157705783844f, -1.484498143196106f, 0.2109839767217636f, -0.8684839606285095f, -1.171597957611084f, -0.9773237109184265f, -0.002813637489452958f, -1.2975432872772217f, 0.3479395806789398f, -0.7253870964050293f, -0.6621626615524292f, -0.2912011444568634f, -2.3239030838012695f, 0.918861985206604f, 1.1420683860778809f, -0.1961705982685089f, -1.5105466842651367f, 0.0809166356921196f, 1.4640601873397827f, -0.4027494490146637f, 1.6002179384231567f, -0.26083904504776f, -0.6423792839050293f, -1.7842353582382202f, -1.4978502988815308f, 1.380509853363037f, 1.877629041671753f, -1.4456785917282104f, -0.28545263409614563f, 0.15021729469299316f, 0.5569579005241394f, -1.6427204608917236f, -0.6389901041984558f, -0.5466802716255188f, -1.0836117267608643f, -1.5072098970413208f, -0.3856680691242218f, 0.9019067287445068f, -0.2348310351371765f, -1.738688588142395f, 0.28999093174934387f, -0.1533069908618927f, -0.9047854542732239f, 0.3279983699321747f, -0.5037038922309875f, -1.9761329889297485f, 0.12128926813602448f, -0.06794015318155289f, 1.1322118043899536f, -1.0514512062072754f, -0.672996997833252f, 0.549104630947113f, -0.32529518008232117f, 0.3177003860473633f, 0.1834784746170044f, 1.7332375049591064f, -0.2832452356815338f, -1.384834885597229f, -0.010661765933036804f, -0.22356857359409332f, 0.5824847221374512f, -0.6592251062393188f, 0.44558772444725037f, -2.2453958988189697f, 0.42556077241897583f, -0.6481807231903076f, -0.25188785791397095f, -1.27206289768219f, 1.1171956062316895f, 1.5053801536560059f, 0.8270988464355469f, 0.7335371971130371f, -0.7486765384674072f, 0.45417457818984985f, -0.3060213625431061f, 0.6338478326797485f, -2.674952745437622f, -0.6763451099395752f, -0.9770270586013794f, 1.5783367156982422f, -0.2843257188796997f, -1.3570187091827393f, -2.2979846000671387f, -0.8282976150512695f, -1.328344464302063f, -0.0004540873342193663f, 0.6817458271980286f, 0.9938647150993347f, -1.4253593683242798f, 0.3879549503326416f, -2.0311243534088135f, 1.2712687253952026f, 0.185286283493042f, 0.07715956121683121f, -0.584597110748291f, -0.5161778330802917f, 0.3677023947238922f, -1.3314623832702637f, 1.5407487154006958f, -0.3439099192619324f, -1.2492866516113281f, -0.8399935364723206f, 0.40630635619163513f, 0.00801965780556202f, -0.3249308466911316f, -0.6664285659790039f, 0.28344160318374634f, 0.5257611274719238f, -1.3125312328338623f, -0.3971971273422241f, -1.5499167442321777f, -0.19803878664970398f, 1.0783737897872925f, 0.5109858512878418f, 0.45442286133766174f, -0.7324386835098267f, -0.1001700609922409f, -0.9963456988334656f, 1.706158995628357f, -0.28345587849617004f, 0.24911366403102875f, -0.5420844554901123f, 0.38604170083999634f, -0.08859143406152725f, -0.7755544781684875f, -0.3989383280277252f, 0.3021530210971832f, 0.8383589386940002f, 0.36416593194007874f, -1.461991786956787f, 0.15854018926620483f, -0.5639234185218811f, 1.251044750213623f, 1.9695827960968018f, 1.165942907333374f, -1.0710505247116089f, -1.4054443836212158f, 2.9302103519439697f, -0.23575513064861298f, 0.08357129991054535f, -0.09462077915668488f, -1.8668535947799683f, -0.34727850556373596f, -0.35426023602485657f, 1.4054851531982422f, 1.4522364139556885f, -0.08649589866399765f, 0.04004557058215141f, -0.10994749516248703f, -0.6241140961647034f, -0.24008060991764069f, -0.39534568786621094f, 0.007234930992126465f, 0.03980718180537224f, -1.2252846956253052f, -0.5507245063781738f, -1.171595811843872f, 1.0313161611557007f, -0.20263050496578217f, 1.3937491178512573f, 1.0930505990982056f, 1.4651625156402588f, 0.2634923756122589f, -0.4779359698295593f, -1.4054735898971558f, 0.7812330722808838f, 0.22484587132930756f, 0.7241055965423584f, -0.6603827476501465f, -0.6927292346954346f, -0.7928091883659363f, -0.4532349705696106f, 0.5121238231658936f, -0.3005983531475067f, -0.44323164224624634f, -0.3902406692504883f, -1.0369137525558472f, -0.19362880289554596f, 0.49349725246429443f, -0.43515950441360474f, -0.6069482564926147f, -1.473711609840393f, -2.3762154579162598f, 1.134980320930481f, -0.9291712641716003f, -0.62174391746521f, 0.47012677788734436f, 1.0591752529144287f, 0.3450745940208435f, -0.04975222051143646f, 1.2706385850906372f, 0.950104296207428f, 0.9791541695594788f, -0.0728548914194107f, 1.3175126314163208f, 0.5881410837173462f, 0.9367974996566772f, 0.9128003120422363f, 0.4015538990497589f, 0.1372641772031784f, 1.041069507598877f, 0.7456277012825012f, 0.0962943509221077f, -1.7508755922317505f, -1.171151041984558f, 1.6682186126708984f, -0.6773730516433716f, -2.147958517074585f, 0.4557051658630371f, 0.061060722917318344f, 0.4539925456047058f, -1.1904879808425903f, 0.2646864056587219f, -0.8139405250549316f, -2.294616460800171f, -0.28324073553085327f, 1.7303272485733032f, -0.46407395601272583f, 1.7161751985549927f, 0.8312178254127502f, 0.2559703290462494f, 0.7086811661720276f, 0.6478610038757324f, -0.06174112856388092f, 0.2679769694805145f, -0.6163668632507324f, -0.4823808968067169f, 0.3113662600517273f, -1.5075656175613403f, -0.32805025577545166f, 0.9543997049331665f, 0.8449776768684387f, -0.8963518738746643f, 0.354059636592865f, -0.15964199602603912f, -0.20036070048809052f, 0.4515315592288971f, 0.987306535243988f, -0.9755827188491821f, 0.2618931531906128f, -2.5518243312835693f, 1.015307068824768f, 2.9806666374206543f, -0.5756134390830994f, -0.42303135991096497f, -1.6033560037612915f, -0.3883368968963623f, -1.6651710271835327f, -1.7486891746520996f, 0.5733898282051086f, -0.17000924050807953f, -1.2242252826690674f, -0.7774103879928589f, -0.662408173084259f, 1.8391462564468384f, 0.4153856635093689f, 0.07132068276405334f, 0.595319926738739f, -0.8092124462127686f, 0.3273322880268097f, -0.4520135819911957f, 0.15657921135425568f, 2.195984125137329f, 0.9013379216194153f, 0.23789522051811218f, -1.9253568649291992f, -0.8211831450462341f, -0.16658450663089752f, -2.5792624950408936f, 1.1836506128311157f, -0.0034865080378949642f, -0.5498691201210022f, 0.3581060767173767f, 0.7652292251586914f, -0.7733491659164429f, 0.6297365427017212f, -0.6815900802612305f, 1.240025281906128f, 0.8412518501281738f, 0.29200538992881775f, -1.805619716644287f, 1.3460670709609985f, 0.02373058721423149f, -0.6370474696159363f, 1.3327691555023193f, -1.63896906375885f, 0.4103865623474121f, 0.25587576627731323f, 1.880447506904602f, 1.9977002143859863f, -1.038170576095581f, 1.879461407661438f, -0.9597489833831787f, -1.338854193687439f, -1.4890133142471313f, 0.9830695986747742f, -0.41451194882392883f, 1.213753581047058f, 0.5297332406044006f, -0.8235613107681274f, 0.3100223243236542f, 0.7781904339790344f, -0.7196370363235474f, 0.8093208074569702f, -0.7462528347969055f, 0.3914867341518402f, 1.1797000169754028f, -2.474013328552246f, 0.43964120745658875f, -0.10444001853466034f, 0.5850116610527039f, -1.6503483057022095f, -1.7112841606140137f, -0.08656994998455048f, -0.3778044879436493f, -0.3667309880256653f, 0.08071848005056381f, -0.865123450756073f, 0.6451484560966492f, -0.5293470621109009f, -1.4819915294647217f, 1.995280146598816f, -0.6921202540397644f, 1.3029942512512207f, -0.1476615071296692f, 0.15694653987884521f, -0.6405912041664124f, -0.8447388410568237f, 0.3645228147506714f, -0.7561046481132507f, 1.417391061782837f, 0.5760944485664368f, 0.13722558319568634f, -0.41890090703964233f, -1.0182898044586182f, 0.448284387588501f, 1.421317458152771f, -0.6346070170402527f, -1.0847539901733398f, -0.5952850580215454f, -0.36421963572502136f, -1.5131834745407104f, -1.9063634872436523f, -1.7777600288391113f, -0.5803453326225281f, -0.6087163090705872f, 0.4910713732242584f, -0.54014652967453f, 1.2756917476654053f, 0.9132878184318542f, -1.101527452468872f, -0.8013262152671814f, -1.9939244985580444f, 1.0690128803253174f, 0.7788016200065613f, 0.35621726512908936f, -0.47195300459861755f, 0.9912806153297424f, -0.21498170495033264f, 1.6797508001327515f, -0.7104259729385376f, 0.06523545831441879f, 0.9159647822380066f, 2.1436846256256104f, 0.4262869358062744f, 0.2694464921951294f, 0.9167301654815674f, 0.6317405700683594f, 0.07572164386510849f, 1.160980463027954f, -0.6461572051048279f, 0.03240058198571205f, -0.7329845428466797f, 0.5573175549507141f, 0.1672649085521698f, -0.30756810307502747f, -0.27301856875419617f, 0.2073671668767929f, 0.1735081523656845f, -0.8468038439750671f, 0.1224641352891922f, 0.020374706014990807f, 1.1085376739501953f, 1.3970056772232056f, -0.5763441920280457f, -1.7790907621383667f, -1.4155789613723755f, -0.11962825059890747f, 1.434804081916809f, -0.6779595613479614f, 0.08594394475221634f, 0.17818263173103333f, 0.8717845678329468f, -0.4843977093696594f, 0.1896892935037613f, 1.0048247575759888f, 0.5574866533279419f, 0.20942160487174988f, -2.0887160301208496f, -1.1161184310913086f, -0.5786607265472412f, 0.8086466789245605f, 0.8664202690124512f, 1.088623046875f, 0.01803939789533615f, 0.968483567237854f, -0.3800842761993408f, 0.3059806525707245f, 0.21581949293613434f, -0.4931473731994629f, -0.15840917825698853f, 0.09949837625026703f, 0.30463987588882446f, 0.9820282459259033f, 2.1544744968414307f, 0.14304663240909576f, -1.460897445678711f, -0.1060129776597023f, -0.37611088156700134f, -0.7917583584785461f, -0.08468178659677505f, 0.8363486528396606f, -1.3825470209121704f, 0.6469754576683044f, 0.5219618082046509f, -0.6058419346809387f, -1.969196081161499f, -0.9246644377708435f, -0.508442759513855f, -1.044813632965088f, 0.353022038936615f, 0.8511560559272766f, -2.0522587299346924f, -0.02700735814869404f, -0.7895724773406982f, -2.2979352474212646f, 1.6485515832901f, -2.076538562774658f, 0.5667335987091064f, 1.3000932931900024f, 0.34126174449920654f, -0.384304404258728f, -0.2445336878299713f, 0.6715037226676941f, -0.3841925263404846f, -0.8124385476112366f, 0.4127030372619629f, 1.1925286054611206f, -1.131065011024475f, 0.3341250419616699f, -0.5277276635169983f, 0.24021567404270172f, -0.18079474568367004f, 0.9263569712638855f, -2.2063262462615967f, 0.6430982351303101f, 1.010899543762207f, -1.5004045963287354f, -0.6776713728904724f, 0.4190317988395691f, -0.1862286925315857f, 0.49416670203208923f, -0.9246350526809692f, 0.5439687371253967f, 1.2584720849990845f, -0.27466103434562683f, -1.1287552118301392f, 0.3993513286113739f, -1.8577489852905273f, 1.882336139678955f, 0.28583550453186035f, -1.3191593885421753f, -0.04691770300269127f, 1.424126148223877f, 0.47822806239128113f, -0.22128069400787354f, -0.16418692469596863f, 1.2590702772140503f, -1.5019725561141968f, -1.2077754735946655f, -0.5713672637939453f, -0.29712751507759094f, -0.08878864347934723f, -1.2249038219451904f, 0.9701720476150513f, 0.02799459546804428f, 2.49943208694458f, -0.5693896412849426f, 0.2992815375328064f, 0.4264316260814667f, -1.8905819654464722f, 0.5416959524154663f, 0.5037822723388672f, 0.06191536784172058f, 0.7051315307617188f, 1.1914323568344116f, 0.7841975092887878f, -0.5143458247184753f, 0.10756650567054749f, 0.03601418063044548f, -1.586909532546997f, -0.36298084259033203f, 0.05457804724574089f, -1.8995989561080933f, -0.4206298589706421f, 1.725341796875f, 0.2990115284919739f, 0.2708130478858948f, 1.202155351638794f, -1.3849509954452515f, 0.37148424983024597f, 1.2834670543670654f, 0.40953299403190613f, 1.5861903429031372f, 2.2006518840789795f, 1.1788570880889893f, 0.04892049357295036f, 0.45504313707351685f, -1.1838287115097046f, 0.2520817816257477f, -1.0377240180969238f, -1.1767886877059937f, 1.5097410678863525f, 0.6526481509208679f, 0.5476107001304626f, -1.4937485456466675f, -0.714340090751648f, -1.4507311582565308f, 0.3059408664703369f, 1.3935821056365967f, -2.0411956310272217f, -1.7664213180541992f, 1.1327776908874512f, 0.7594925165176392f, 0.8204308748245239f, 0.7248584032058716f, 0.6390634179115295f, -0.5571966767311096f, -0.40140989422798157f, -0.22562873363494873f, -0.5097048878669739f, 0.9061921834945679f, -0.15940190851688385f, -0.4242385923862457f, -0.23000852763652802f, -0.5473349094390869f, 0.3962896168231964f, -1.0022584199905396f, -1.9527696371078491f, -0.9625334739685059f, 0.7991358637809753f, -0.010661646723747253f, 0.023252639919519424f, 0.6258861422538757f, 0.3616181015968323f, -0.2700897455215454f, 1.6419692039489746f, -0.7898518443107605f, -0.32855224609375f, 0.9272356629371643f, 0.23767738044261932f, 2.069274663925171f, -0.6138983368873596f, 1.5534266233444214f, 0.562792956829071f, 0.9539335370063782f, 0.34314191341400146f, 0.32635602355003357f, -1.0357731580734253f, -0.021849583834409714f, 0.4444722533226013f, 1.132546067237854f, -1.3696054220199585f, -0.42242521047592163f, 0.6405972242355347f, 0.026323553174734116f, -1.8021960258483887f, -0.17883431911468506f, 0.6420789957046509f, 0.17955924570560455f, -0.2522568106651306f, -1.4123497009277344f, -0.7108830809593201f, -0.39913156628608704f, 0.6917201280593872f, -1.742424488067627f, 0.37175261974334717f, 1.2827181816101074f, -0.1333233267068863f, 2.0086865425109863f, -0.2341606765985489f, 0.7056452035903931f, 0.3473411798477173f, 0.809699535369873f, -3.0800278186798096f, -1.1150798797607422f, 1.4471967220306396f, 0.812851071357727f, 0.4065439999103546f, -0.545810341835022f, -0.6789452433586121f, -0.7672881484031677f, 0.17289385199546814f, 0.40247759222984314f, 0.6123214364051819f, -0.6715307831764221f, 0.18855510652065277f, 1.0615094900131226f, -0.2978079319000244f, -0.4801141619682312f, -1.3616151809692383f, -0.9356100559234619f, 0.978323757648468f, 1.4345933198928833f, 0.956223726272583f, -0.9492759704589844f, -0.9247362613677979f, 1.4304224252700806f, -1.2141680717468262f, 1.5573612451553345f, -2.772749185562134f, -2.07552170753479f, 0.019730882719159126f, -1.6115124225616455f, 0.2347842901945114f, -0.6375554203987122f, -0.2960043251514435f, -0.41874298453330994f, -1.3194098472595215f, 0.05774686485528946f, 0.30748531222343445f, -0.5826377272605896f, -0.31068262457847595f, 0.39234694838523865f, 1.9441745281219482f, 2.0378758907318115f, 0.4360438287258148f, -0.09107904881238937f, -0.24155908823013306f, 0.45299890637397766f, -0.10979556292295456f, 1.179660439491272f, 1.1283513307571411f, -0.8159404993057251f, -0.07127540558576584f, -0.8951465487480164f, 0.31810882687568665f, 0.6084815859794617f, 0.19665859639644623f, -1.3970006704330444f, 1.1369043588638306f, -0.11328133195638657f, 0.12461294233798981f, -0.5991875529289246f, -1.8511996269226074f, 1.1412599086761475f, -1.323111653327942f, -1.642134666442871f, -0.7886322736740112f, 0.016923261806368828f, 0.26853591203689575f, -0.08590231090784073f, -1.8297702074050903f, 1.8586663007736206f, 0.6306189298629761f, -2.0020906925201416f, -0.9767837524414062f, -1.0065470933914185f, 1.5179888010025024f, -0.25139299035072327f, 0.2684777081012726f, 0.2379687875509262f, -1.7342220544815063f, 0.890319287776947f, -0.038704462349414825f, -1.1842435598373413f, -0.6990516185760498f, -0.6046251654624939f, 1.8508250713348389f, -2.0493991374969482f, -1.7228929996490479f, 0.23755493760108948f, 0.9717959761619568f, -0.3798253536224365f, 0.685245931148529f, -1.1771284341812134f, -0.02216072380542755f, -0.1269814372062683f, -0.26579511165618896f, 0.22640825808048248f, -0.760986864566803f, 0.4831203520298004f, 1.5869791507720947f, -0.6875012516975403f, -1.2886475324630737f, 1.2204482555389404f, -0.5210925936698914f, 2.4901089668273926f, 0.15757296979427338f, -0.6784350275993347f, 1.6848376989364624f, -0.762600839138031f, 0.4223840832710266f, 0.05342012271285057f, -1.407547950744629f, 0.7069153189659119f, 0.5444626212120056f, -0.19080407917499542f, 0.14213266968727112f, 0.20559696853160858f, 1.2673413753509521f, 1.375124216079712f, 0.060717396438121796f, 0.31265851855278015f, -0.1293685883283615f, -1.0525683164596558f, -0.5546149015426636f, 1.0222612619400024f, -0.8595789074897766f, 1.0922929048538208f, -0.7460381984710693f, 0.660965621471405f, 0.454408198595047f, -1.4719254970550537f, 0.376058965921402f, -0.7192203998565674f, -0.9688137173652649f, 0.4969656765460968f, 0.42114102840423584f, -0.3883015811443329f, 0.9459248185157776f, 0.22322431206703186f, -0.3906600773334503f, 0.5856817960739136f, 0.2909909188747406f, -1.3024108409881592f, 1.4306825399398804f, 0.4534458816051483f, 0.9106372594833374f, -0.341061532497406f, -0.7916259765625f, -1.1452873945236206f, -0.4609397351741791f, -0.5050092339515686f, 1.0576668977737427f, 0.8928799033164978f, 0.7486283779144287f, 1.0225255489349365f, 1.0325298309326172f, 0.772061824798584f, 0.6529440879821777f, 1.3521305322647095f, 1.9143685102462769f, -1.3935517072677612f, 0.9990993142127991f, 1.297232985496521f, 2.667762517929077f, 0.888896107673645f, 0.8407386541366577f, 1.3103235960006714f, 1.3254046440124512f, 0.7249560952186584f, 1.3489890098571777f, 0.7757742404937744f, 0.16486017405986786f, -1.9583953619003296f, 2.0979738235473633f, -1.1749324798583984f, -1.5510687828063965f, 0.6880179643630981f, -0.27131080627441406f, -2.00603985786438f, -0.14031292498111725f, 0.24766799807548523f, -0.19394119083881378f, -0.08344011753797531f, 1.1331552267074585f, 0.4335167407989502f, 1.659290075302124f, 0.645967423915863f, -0.5915296077728271f, 1.6242163181304932f, -0.9242795705795288f, 1.5664156675338745f, -1.430296540260315f, 0.24254998564720154f, -0.9536606073379517f, 0.2823708951473236f, 0.9416194558143616f, 0.9761032462120056f, -0.14037710428237915f, 0.21578916907310486f, 0.42492198944091797f, 0.9744104146957397f, -0.6820825934410095f, 0.7403110861778259f, -0.014602077193558216f, -1.9035120010375977f, -0.459785133600235f, -0.22833675146102905f, 0.2052573710680008f, 0.060564182698726654f, -0.6736571192741394f, -0.811122477054596f, -0.011842876672744751f}; - -// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g) -// = Bpack_dram + (nt*(K/4)+g)*64. -static float Bpack_dram[64*64] __attribute__((section(".data"))) = -{-0.5196930766105652f, 1.8524175882339478f, 1.8365377187728882f, 2.074131727218628f, -0.7373097538948059f, -0.7686780691146851f, -0.05119974538683891f, 1.5985578298568726f, 0.2122737318277359f, 1.1059595346450806f, 1.311963438987732f, 0.424176424741745f, -0.4922901690006256f, 1.6562608480453491f, 0.4111401438713074f, -0.2428770512342453f, 2.081841230392456f, 1.0677173137664795f, -1.4276772737503052f, -0.33180344104766846f, 1.7054011821746826f, 0.6059234738349915f, 1.1122153997421265f, -0.5635794997215271f, -1.364527702331543f, 0.17917323112487793f, 0.5652397871017456f, 0.3271985352039337f, 0.01357425469905138f, 2.691838502883911f, 1.2729166746139526f, -0.6343013048171997f, -0.10098669677972794f, -0.43878018856048584f, -1.9775099754333496f, -0.03591495007276535f, -0.787344753742218f, -0.11653880029916763f, 1.899356484413147f, 0.45238569378852844f, -0.5850009918212891f, 0.17411360144615173f, -0.2031504362821579f, -1.271364688873291f, 0.8729038834571838f, 0.14136412739753723f, -1.4058030843734741f, 0.4083256125450134f, -1.3799183368682861f, -0.21034009754657745f, 0.8919094204902649f, 0.18319325149059296f, 0.5042280554771423f, -1.007485270500183f, -1.3393090963363647f, -0.4502287805080414f, -0.16011619567871094f, -1.608611822128296f, 0.1806594282388687f, 1.864193320274353f, -1.201043963432312f, -0.07190026342868805f, 0.14386875927448273f, -0.1494988203048706f, 0.6289454698562622f, -0.0685209259390831f, -1.2115036249160767f, -0.5285655856132507f, -1.5568547248840332f, -1.2001843452453613f, 0.2736855447292328f, -0.6129935383796692f, 0.9846591353416443f, 1.6213051080703735f, 0.5066256523132324f, -0.126494362950325f, 0.03450252488255501f, 1.3147404193878174f, -0.6877241134643555f, -0.10271778702735901f, -1.401864767074585f, -0.134507417678833f, -0.5799044966697693f, 1.0302584171295166f, 0.16683164238929749f, 0.13005995750427246f, 0.3381371796131134f, 1.025931715965271f, -1.1549469232559204f, 0.21058911085128784f, 0.0752936601638794f, 0.4396262466907501f, 0.029413584619760513f, 0.18214739859104156f, 0.03471093624830246f, -1.8955014944076538f, 0.9624823927879333f, 0.04533285275101662f, -2.03769850730896f, -1.0922635793685913f, -0.17383840680122375f, -1.065432071685791f, 0.926749587059021f, -0.16234397888183594f, 0.36853712797164917f, -0.2603956162929535f, 0.12695534527301788f, -1.7026076316833496f, 0.6422180533409119f, -0.6823133230209351f, -1.6458345651626587f, 0.15697401762008667f, -1.2265124320983887f, 0.7861263155937195f, -2.095715284347534f, 0.4538240432739258f, -0.4390734136104584f, 0.4836112856864929f, 0.9263717532157898f, 0.06930986046791077f, 1.4818024635314941f, -0.9132123589515686f, -1.606928825378418f, -0.5966755747795105f, 0.7912061810493469f, 1.1138908863067627f, 1.4326399564743042f, 0.34999290108680725f, 0.7664128541946411f, 0.4324735403060913f, -1.018404245376587f, 0.012393372133374214f, 0.6695152521133423f, 1.2033629417419434f, 0.8298293352127075f, -0.9191604256629944f, 0.958231508731842f, 0.2724153697490692f, 0.9915789365768433f, 0.3825709819793701f, -0.5029784440994263f, 1.5228251218795776f, -2.502131938934326f, -0.6252055764198303f, -0.5682758688926697f, -1.4246463775634766f, 0.4983586370944977f, -1.3945032358169556f, -2.804393768310547f, 0.4811320900917053f, 0.3296896517276764f, 1.4936070442199707f, 1.691656231880188f, 0.1595890074968338f, 1.024785041809082f, 0.12853768467903137f, -0.46378275752067566f, 1.3842717409133911f, -0.6843701601028442f, -0.3655944764614105f, -0.34340038895606995f, 0.9877837896347046f, 1.3732529878616333f, -0.32952919602394104f, -0.540728747844696f, 0.5175154209136963f, -1.1346107721328735f, 0.4711856544017792f, 1.920310378074646f, 0.4751400351524353f, 0.5673654675483704f, 0.2462792843580246f, 2.888267755508423f, -0.5556330680847168f, 0.017246929928660393f, -0.3958292305469513f, -0.3625917434692383f, 1.179726004600525f, 0.8268131613731384f, 1.457729697227478f, -1.029435634613037f, -1.2957319021224976f, 0.2842133045196533f, -2.20194673538208f, 0.5491127967834473f, -1.2930694818496704f, 0.4510805904865265f, -0.3953653573989868f, 1.7161552906036377f, 0.15392881631851196f, -1.4651801586151123f, -0.5170696377754211f, -0.789655864238739f, -1.313944935798645f, 1.211775779724121f, 0.9332025647163391f, -0.2605237364768982f, 1.5151708126068115f, -1.521173357963562f, -1.1669979095458984f, -2.0334060192108154f, -0.08260460197925568f, 1.4479997158050537f, 0.1535591334104538f, -0.19014781713485718f, -0.9898110032081604f, 0.3790753483772278f, 1.921068549156189f, -0.13450416922569275f, 0.5259649753570557f, 1.2340812683105469f, 0.19075801968574524f, -0.19310960173606873f, 1.5767059326171875f, 0.11904352903366089f, 0.2899794280529022f, -2.0928122997283936f, -0.0447654202580452f, 0.11111800372600555f, 0.38851943612098694f, 0.5478768348693848f, -1.3495665788650513f, -0.44743525981903076f, -0.7099960446357727f, 0.9817765355110168f, 1.0983757972717285f, -0.25671127438545227f, -2.2621774673461914f, 0.16864269971847534f, -1.6143133640289307f, -0.011093960143625736f, 1.2098065614700317f, -1.4476906061172485f, 1.3612878322601318f, -1.3062708377838135f, 1.6407432556152344f, -0.33931660652160645f, 0.13347899913787842f, -0.9141297936439514f, -0.15479597449302673f, -0.699771523475647f, -1.096972107887268f, -0.55166095495224f, -0.3263216018676758f, 0.6205294132232666f, -1.4098610877990723f, -0.41539478302001953f, 3.068681478500366f, 0.3701395094394684f, -0.7284356355667114f, -0.9289583563804626f, 0.8983376026153564f, -0.48385927081108093f, 0.032833296805620193f, -0.10948953032493591f, 0.46667027473449707f, 0.6613953113555908f, -0.03297487273812294f, -0.13994985818862915f, 1.0257868766784668f, -1.9717178344726562f, 0.5998077988624573f, 1.361150860786438f, -0.173064723610878f, 0.7555463910102844f, -0.14636565744876862f, -0.09429822862148285f, 0.29657915234565735f, 0.11924134939908981f, 0.1167483851313591f, -0.5673035979270935f, -1.7532938718795776f, -1.0694472789764404f, 0.08452916890382767f, 0.33887022733688354f, -1.223060965538025f, 0.19821691513061523f, 0.38515371084213257f, -0.07315804064273834f, 0.8780375719070435f, 0.12963341176509857f, -0.6663824319839478f, 0.4921484887599945f, -0.9834228754043579f, -1.2953417301177979f, -1.5002580881118774f, 1.1558160781860352f, -0.7528656125068665f, -1.160732388496399f, -1.486362099647522f, 0.13496126234531403f, 1.354621410369873f, 1.9504122734069824f, 1.0814120769500732f, 0.7843427658081055f, 1.8615505695343018f, 0.6727956533432007f, -0.4322131276130676f, 1.8005059957504272f, 1.3338027000427246f, -0.3184575140476227f, 1.1182889938354492f, 1.8907326459884644f, 0.19988827407360077f, 0.9396787285804749f, 1.0826958417892456f, -0.6519155502319336f, 0.2863307297229767f, 0.10145679116249084f, 0.09791278839111328f, -0.8082833886146545f, 1.441809892654419f, -0.6460452675819397f, 0.04811352491378784f, 1.382570505142212f, -0.6665758490562439f, 0.7876792550086975f, -0.15957334637641907f, -0.10303135216236115f, 0.27407389879226685f, -0.5971476435661316f, -1.0348221063613892f, 0.42101818323135376f, -0.548947811126709f, -1.2540193796157837f, -1.2696188688278198f, 0.6492337584495544f, 0.2340298593044281f, -0.1553155928850174f, -0.4344737231731415f, -1.0962450504302979f, -1.122981309890747f, -1.865479826927185f, -2.1207547187805176f, -0.6719433665275574f, -0.7559810280799866f, 0.1943359375f, 1.0423648357391357f, -1.1257092952728271f, -0.04083564877510071f, 0.7159355282783508f, 0.3418067395687103f, -0.6778577566146851f, 0.17783714830875397f, -0.16209769248962402f, -0.026765741407871246f, -0.33800795674324036f, 0.6160297393798828f, -1.5755101442337036f, -1.1856271028518677f, 0.3281179666519165f, -0.03522142022848129f, -0.5061670541763306f, 0.8149546980857849f, 0.6836134195327759f, -1.2236212491989136f, 0.9091914892196655f, -0.6015433669090271f, 0.8525826930999756f, -0.6866192817687988f, 1.1305443048477173f, -0.3458130359649658f, -0.673958420753479f, -0.16243989765644073f, 1.8315014839172363f, -0.7265217900276184f, -0.44100794196128845f, -1.1563644409179688f, -1.1872515678405762f, -0.83109050989151f, -1.1144185066223145f, 1.2617039680480957f, 0.6814237833023071f, 2.0168917179107666f, 0.481290727853775f, 0.313506156206131f, -0.5229234099388123f, -0.26092126965522766f, -2.14499831199646f, 1.0247118473052979f, 2.1447153091430664f, -0.34384623169898987f, 0.9131002426147461f, -2.4607415199279785f, 1.2518025636672974f, -0.35988521575927734f, -0.43994462490081787f, 1.9879144430160522f, -0.692777693271637f, 0.4603208601474762f, 0.6009213924407959f, 0.8063201308250427f, 0.979621946811676f, 0.9170622229576111f, -0.11666277050971985f, 0.2106197327375412f, 0.5228466987609863f, -1.7686034440994263f, -0.8117372393608093f, -0.877277135848999f, 0.9701831936836243f, -1.7244060039520264f, 1.61734139919281f, -0.9298601746559143f, 0.41942062973976135f, 2.0680532455444336f, -1.2342560291290283f, 2.678502321243286f, 0.3004297614097595f, -1.303833246231079f, 0.7512189745903015f, 0.3729371726512909f, 0.17139992117881775f, 0.5995815396308899f, -0.19726276397705078f, -0.11891679465770721f, -1.0388710498809814f, -0.1431705802679062f, 1.1298640966415405f, 0.587632417678833f, 0.020525068044662476f, 0.12478796392679214f, 0.2550857663154602f, -0.4017896056175232f, -1.7806396484375f, -0.08095061779022217f, -1.759868860244751f, 0.820906937122345f, -0.15008316934108734f, -1.8686583042144775f, -0.6503500938415527f, 0.6934462189674377f, 1.4084579944610596f, -2.2690858840942383f, 0.39256006479263306f, -0.4266415536403656f, 0.36491602659225464f, 0.807181715965271f, -1.5602378845214844f, -0.055706802755594254f, -0.712527871131897f, 0.2629498243331909f, 1.1325268745422363f, 0.8828094601631165f, 0.9763681888580322f, 0.35255447030067444f, -0.5103265047073364f, 0.048219867050647736f, -0.8320447206497192f, 0.9133552312850952f, 0.35530006885528564f, 1.5981814861297607f, 0.6848441362380981f, -0.238090381026268f, -0.8003812432289124f, 0.22974559664726257f, -0.19079573452472687f, -0.364820271730423f, -0.658343493938446f, -1.2190016508102417f, -1.0552600622177124f, -0.9370644688606262f, 0.07260560989379883f, 0.24500113725662231f, 1.690338134765625f, 1.8711999654769897f, 1.2940467596054077f, 0.49941954016685486f, -0.5331865549087524f, -0.26645612716674805f, 0.3932536244392395f, 0.45422324538230896f, 0.4487703740596771f, -0.43473342061042786f, 1.8583357334136963f, 0.35827821493148804f, 0.1845020353794098f, 1.472187876701355f, 0.9056950807571411f, -2.0051839351654053f, -1.232080101966858f, 1.510047435760498f, -0.4955300986766815f, -1.451595664024353f, 1.1304244995117188f, -0.23605898022651672f, -1.4099090099334717f, 0.033967722207307816f, -0.27385643124580383f, 0.052069131284952164f, 0.6194980144500732f, 1.2993793487548828f, 0.37411054968833923f, 0.29060807824134827f, -0.14929574728012085f, 0.16823826730251312f, 1.7945916652679443f, -0.8427640199661255f, -0.776123583316803f, 0.7065202593803406f, 0.020995743572711945f, -0.7148111462593079f, -0.21817894279956818f, 0.6545652151107788f, -1.3288897275924683f, -0.5552407503128052f, 0.869994044303894f, 0.17624177038669586f, 0.10479936003684998f, 1.1782784461975098f, -0.8539279699325562f, 0.18701297044754028f, -0.3077951669692993f, 1.3164128065109253f, 0.7719241976737976f, 2.0046799182891846f, -0.05154263228178024f, -0.09019286930561066f, 0.9585400819778442f, -1.2275644540786743f, 0.35432279109954834f, -0.2604968845844269f, 0.6541568636894226f, -0.43890517950057983f, -0.05930287018418312f, -1.202606201171875f, -1.2769453525543213f, -0.3968345820903778f, 0.5857738256454468f, -0.3441990613937378f, 0.6814432144165039f, 0.772460401058197f, 0.007647526450455189f, -0.25153985619544983f, 0.84389328956604f, -0.2621386647224426f, -0.42434096336364746f, -0.6080525517463684f, 0.011438353918492794f, 0.0011970907216891646f, 0.672935426235199f, -0.6712173819541931f, -0.2299986481666565f, 0.13960936665534973f, 0.6251678466796875f, -1.8366930484771729f, -0.45854899287223816f, 0.49610286951065063f, -0.8524967432022095f, -0.19241304695606232f, 0.47062209248542786f, -0.7075293660163879f, 0.01930052787065506f, 0.7421151995658875f, -2.1299924850463867f, -1.8519654273986816f, -0.789726972579956f, -1.3833003044128418f, -1.1265941858291626f, 0.4917159378528595f, 0.744454026222229f, -0.761397659778595f, 1.7994297742843628f, 0.6710255146026611f, 1.5392361879348755f, -0.8462389707565308f, -1.374268651008606f, -0.29935309290885925f, -1.3537677526474f, -0.9374998211860657f, 0.966241180896759f, 2.2905983924865723f, 0.4570314586162567f, 1.168905258178711f, -0.22257985174655914f, 1.2293546199798584f, -0.4278397262096405f, -0.527760922908783f, -1.798736572265625f, -0.07677789032459259f, 0.781619668006897f, 0.13910506665706635f, -1.7339553833007812f, 0.7391383647918701f, 1.4115384817123413f, 0.38143935799598694f, -0.051372598856687546f, 0.867921769618988f, -1.1712745428085327f, 1.219971776008606f, -0.9404910206794739f, 0.6608859300613403f, 0.49976596236228943f, 0.9063552021980286f, -0.273310124874115f, -0.8444989919662476f, 0.06740748137235641f, -1.1771124601364136f, -2.319305896759033f, 1.0881538391113281f, 1.7532049417495728f, -1.352665662765503f, -1.717094898223877f, 0.539482057094574f, -0.4647720456123352f, 0.15511354804039001f, -0.933184802532196f, 2.0511553287506104f, 1.7320348024368286f, 0.39241987466812134f, -0.3704788386821747f, 0.8811249136924744f, 0.537391722202301f, -0.48424550890922546f, -1.1632671356201172f, 0.3275741636753082f, -1.6853573322296143f, -1.058225393295288f, 1.4543863534927368f, -0.2975543737411499f, -0.8741039633750916f, -1.0332050323486328f, -0.41522952914237976f, 1.5585501194000244f, 2.5905730724334717f, -0.14419972896575928f, -0.47694188356399536f, -0.2289033830165863f, 0.3704476058483124f, 1.024267315864563f, 0.35092905163764954f, -0.11602573096752167f, -0.5128945112228394f, 0.8613727688789368f, 2.617750883102417f, 0.8522545695304871f, 0.3442921042442322f, -0.21941426396369934f, 1.6952100992202759f, 0.7312605381011963f, -2.2600386142730713f, -1.3617544174194336f, 1.5765711069107056f, 0.1465187966823578f, 1.8045871257781982f, 1.3033205270767212f, 0.18405571579933167f, -1.9564915895462036f, 0.1487964242696762f, -0.9366428256034851f, -0.950509250164032f, -0.2993629574775696f, -0.6517760157585144f, 0.2768969237804413f, -0.006574048660695553f, -1.8791542053222656f, -1.5537861585617065f, -0.9000288248062134f, 1.180068850517273f, -1.0849788188934326f, 0.6662015318870544f, -0.7253139615058899f, 1.3528261184692383f, -0.1364641636610031f, 1.1062331199645996f, -0.2775273025035858f, -0.7301269769668579f, -0.9058868288993835f, 0.3395512104034424f, 0.954081654548645f, -0.042208630591630936f, 0.05911993607878685f, -0.3268890678882599f, 0.8507034778594971f, -0.18483993411064148f, -1.1319291591644287f, -0.03408394008874893f, 1.5095572471618652f, -0.11943084001541138f, 1.4642930030822754f, 0.8833335041999817f, 1.1377960443496704f, 0.7793198823928833f, 0.07684005051851273f, -1.4407286643981934f, 0.5401078462600708f, -0.13687263429164886f, -0.4707719087600708f, -0.23642085492610931f, 2.775019645690918f, -0.048772864043712616f, 0.9577808976173401f, -0.12049184739589691f, -0.008940046653151512f, -1.624023199081421f, -0.5534864068031311f, 0.8154556751251221f, -0.03924920782446861f, 1.350463628768921f, -0.04840381070971489f, -0.7114460468292236f, -0.7299661040306091f, -1.4113084077835083f, 0.34884995222091675f, -0.43344026803970337f, 2.270435094833374f, 2.0011656284332275f, -2.1169581413269043f, 0.3513871729373932f, -0.9149423837661743f, -0.28648310899734497f, 1.87372624874115f, 0.07205427438020706f, 0.0007797196158207953f, -0.14003179967403412f, 0.9105306267738342f, -1.0649497509002686f, 0.45754849910736084f, 0.15436618030071259f, -0.744165301322937f, -0.51121985912323f, 0.23932333290576935f, 0.9544809460639954f, 0.16341786086559296f, -0.30999821424484253f, 0.15972714126110077f, 0.8302736878395081f, -1.421786904335022f, -0.09449474513530731f, 0.07646692544221878f, 0.15851546823978424f, -0.38410186767578125f, -1.4129188060760498f, -1.0182603597640991f, 0.4253365993499756f, 0.98757004737854f, 1.1435766220092773f, 0.042469993233680725f, -0.12866179645061493f, -0.3201303780078888f, 1.3020575046539307f, 0.05820586159825325f, -1.411534309387207f, -0.5787767171859741f, -1.1051533222198486f, 1.3300304412841797f, -0.41862916946411133f, -0.43831324577331543f, 1.0452755689620972f, 0.7565826773643494f, 1.3442822694778442f, -0.005769374314695597f, 1.3893702030181885f, -1.459441065788269f, 0.7733556032180786f, 1.281721591949463f, 1.922197699546814f, 1.5146862268447876f, -0.9400398135185242f, 1.7596426010131836f, 0.4703059196472168f, -1.1514455080032349f, -0.6767635941505432f, 0.9793415665626526f, 0.7851172089576721f, 0.9229853749275208f, -1.3558963537216187f, -0.4045674800872803f, -0.022665904834866524f, 0.04516039043664932f, -0.0492362417280674f, 0.688543975353241f, -0.6404033303260803f, -0.8518562912940979f, 0.5739071369171143f, -1.0256696939468384f, 0.5530444383621216f, 1.2585986852645874f, 0.23174597322940826f, -1.7316545248031616f, -0.300735741853714f, 0.8933703303337097f, -0.2885432839393616f, -0.7255284190177917f, 0.17296810448169708f, -1.597967267036438f, -0.8826010227203369f, -0.7285105586051941f, 1.0816147327423096f, 0.689936101436615f, 0.1465923935174942f, -0.2598642110824585f, 1.2287657260894775f, 0.4021441638469696f, 0.29235512018203735f, -1.4843645095825195f, -1.288714051246643f, 0.8229716420173645f, -1.8816033601760864f, -0.24393519759178162f, 1.1026350259780884f, 0.12240499258041382f, -0.6282588839530945f, -0.6169579029083252f, 0.8369725942611694f, -0.21276021003723145f, -1.037753939628601f, -0.5319015383720398f, -1.4479600191116333f, -0.7402247190475464f, -1.9904698133468628f, -0.8090559244155884f, -2.649178981781006f, 0.6452047228813171f, 1.6631805896759033f, -0.6097846031188965f, -0.5651611685752869f, -0.09456146508455276f, 0.2765144407749176f, 0.2960791289806366f, 1.2084519863128662f, -2.0801830291748047f, 0.8232543468475342f, -0.5484461784362793f, 0.34564146399497986f, 1.220015525817871f, -0.43524736166000366f, -1.7363234758377075f, -1.441352128982544f, 1.4536579847335815f, 1.2042756080627441f, -0.9513401389122009f, 1.7534879446029663f, -1.5643774271011353f, 0.2642883360385895f, 0.5893656611442566f, -0.6958070397377014f, -0.9822604656219482f, 0.9073857069015503f, 0.13120052218437195f, 0.3665435016155243f, 0.8943644165992737f, 0.990159809589386f, 1.0685539245605469f, 0.88179612159729f, 0.11932405829429626f, -1.6899914741516113f, -0.6570671200752258f, 1.6827791929244995f, 0.4652342200279236f, -0.9479421973228455f, -0.527360200881958f, -0.17596317827701569f, 1.684232473373413f, -0.005598508287221193f, -1.2838374376296997f, 0.16975225508213043f, 0.8452434539794922f, 0.28028908371925354f, 0.920665442943573f, 1.9825818538665771f, -1.0555790662765503f, -0.7873882055282593f, -0.31694626808166504f, -0.1027197390794754f, -1.1638461351394653f, -0.17423255741596222f, -0.9778326153755188f, 0.07384445518255234f, 0.2161293476819992f, -2.4143142700195312f, 0.6280608177185059f, 0.604034423828125f, -0.06171112880110741f, -0.27243953943252563f, 2.0528199672698975f, -0.6635909676551819f, 0.4586471915245056f, -1.6872135400772095f, 0.03186811879277229f, 2.094301462173462f, -0.38392743468284607f, 0.5313023328781128f, 2.5650343894958496f, -0.06663789600133896f, -0.08388976007699966f, 0.5069190263748169f, -0.9045271277427673f, -0.13790902495384216f, 1.3361674547195435f, 1.065782904624939f, 0.9382249712944031f, -0.5762946009635925f, 0.26595842838287354f, -0.515076756477356f, 1.8256999254226685f, -0.057207878679037094f, -0.2434438019990921f, 0.9149488806724548f, -2.118633508682251f, -1.2563103437423706f, 1.8411842584609985f, 0.8629192113876343f, -1.979547142982483f, 0.4109382629394531f, 0.054416440427303314f, 0.24019105732440948f, -0.3381706476211548f, -1.1024436950683594f, 0.2338137924671173f, -0.4720325171947479f, -1.3284554481506348f, 0.5184157490730286f, -0.48354241251945496f, 1.3702332973480225f, 0.18823130428791046f, -1.2953920364379883f, -0.13154013454914093f, 0.08959710597991943f, -0.44053706526756287f, 0.20576980710029602f, -1.0061860084533691f, 1.1124646663665771f, -0.7451666593551636f, -0.9709926247596741f, -0.005489187315106392f, 0.8581770658493042f, 0.8032210469245911f, -0.15868736803531647f, 1.6983613967895508f, -0.055956143885850906f, -0.45469725131988525f, 1.2942347526550293f, 0.1953326016664505f, -1.775277853012085f, 1.508985161781311f, -0.47618457674980164f, 1.1132714748382568f, 2.190094470977783f, 0.3530663549900055f, 0.4473947584629059f, 0.7191661596298218f, -1.9299876689910889f, 0.9975112676620483f, 1.1845413446426392f, 1.3882135152816772f, 0.1371254324913025f, -0.84199458360672f, -1.6440988779067993f, -0.06394191086292267f, -0.201401948928833f, 0.2995571792125702f, -1.3531256914138794f, 0.8332006335258484f, -0.1304839849472046f, 0.5836395621299744f, -0.38578328490257263f, -1.6579194068908691f, -0.2272949367761612f, 0.6312239170074463f, -1.346675992012024f, 0.6521760821342468f, -1.3508291244506836f, 2.1090853214263916f, -0.8325138688087463f, -0.9411794543266296f, -0.8664024472236633f, 0.3974232077598572f, 0.33468785881996155f, 0.23106975853443146f, -1.2114311456680298f, -1.2591102123260498f, -0.010216501541435719f, 0.9779312610626221f, 0.0748591348528862f, -1.8819596767425537f, 0.27785420417785645f, 0.734195351600647f, -0.37356483936309814f, -0.395193487405777f, -1.2448596954345703f, -0.4259602427482605f, -0.9261072874069214f, 0.33485373854637146f, 1.0248674154281616f, -0.6913897395133972f, -0.5325083136558533f, -2.0061066150665283f, 0.5374981760978699f, 0.4246957302093506f, -0.39820799231529236f, 0.4305093586444855f, 0.7205038070678711f, -0.2832542359828949f, 0.09373100101947784f, 1.657375454902649f, 1.129982829093933f, 0.6332632303237915f, 0.23538991808891296f, 0.08460769802331924f, 1.6273547410964966f, 1.4943406581878662f, 0.4516901969909668f, 0.681415319442749f, -2.738783597946167f, -0.04374435544013977f, -0.13210587203502655f, -0.20210233330726624f, -1.3241872787475586f, -0.1972821205854416f, 0.29198724031448364f, -1.2132630348205566f, 0.7370628118515015f, 0.9276220798492432f, 0.6291629076004028f, -0.23619088530540466f, -0.02293848805129528f, -2.780918836593628f, -0.8419364094734192f, -0.6721378564834595f, 0.13255943357944489f, -1.203613042831421f, 1.333530068397522f, -2.03551983833313f, 0.8634518384933472f, -1.4491990804672241f, 0.3142701983451843f, -1.005286455154419f, -1.3435431718826294f, 1.2677130699157715f, -1.2937111854553223f, -0.7414584755897522f, -0.3299030065536499f, 0.3301123380661011f, 0.9814369082450867f, -1.4912174940109253f, 0.5385298132896423f, 1.3361884355545044f, -0.5637743473052979f, 0.663472592830658f, 0.5684458613395691f, 0.5110347270965576f, 0.9753285646438599f, 1.9773973226547241f, -1.2740811109542847f, -0.7873809337615967f, 2.1380560398101807f, 0.3426303565502167f, 0.9504527449607849f, -1.5345426797866821f, -0.17094235122203827f, 0.9801839590072632f, -0.05862395092844963f, 2.073120594024658f, -0.3976811170578003f, -0.20158079266548157f, 1.4582887887954712f, -0.5315611362457275f, 0.8970864415168762f, -1.1392240524291992f, -1.309956669807434f, -0.0961947962641716f, -2.4108095169067383f, -0.009612545371055603f, -0.6612817049026489f, 0.8028563857078552f, -0.0704915001988411f, -1.5173195600509644f, 0.20034420490264893f, -0.24687924981117249f, 0.23627454042434692f, 0.7618011236190796f, -0.4506336748600006f, 0.3065869212150574f, 0.9810793399810791f, -0.5398741960525513f, -0.42057791352272034f, 0.4069875180721283f, -0.3191024363040924f, 0.8826714158058167f, 0.6646241545677185f, -2.1909663677215576f, 0.4844568967819214f, 0.6883501410484314f, 1.6444298028945923f, -0.09725860506296158f, -0.972576379776001f, 0.9053502678871155f, 0.3767105042934418f, 0.6639789342880249f, 0.05910481512546539f, 0.6989551782608032f, -0.16035781800746918f, -1.092200756072998f, 0.02478332258760929f, 0.4938628077507019f, -0.5733305811882019f, 0.6898083090782166f, 0.6767069697380066f, 0.5416850447654724f, -0.17832504212856293f, -0.7599223852157593f, 0.4881221354007721f, -1.4561625719070435f, 0.5495442748069763f, 0.1979023516178131f, -0.33105704188346863f, -1.3217003345489502f, -0.24350062012672424f, -0.4956028163433075f, 1.6506794691085815f, 0.7990935444831848f, 0.6958364844322205f, -0.9322998523712158f, 0.18523266911506653f, 0.039212681353092194f, -1.0234346389770508f, 0.31960463523864746f, -0.8433935046195984f, -2.1094107627868652f, -0.36244451999664307f, -0.12838764488697052f, -0.30779823660850525f, -1.713757038116455f, 0.23439815640449524f, -0.40255531668663025f, -1.402301549911499f, -0.8660640120506287f, 0.8337363600730896f, 0.1786288470029831f, 1.002273440361023f, -0.676571249961853f, 0.02751591056585312f, -1.0370279550552368f, 1.4250038862228394f, 0.3661484718322754f, 0.2926573157310486f, 0.9981994032859802f, 0.9345365762710571f, 0.08568228036165237f, -0.5905015468597412f, -0.603374183177948f, 0.6896741390228271f, 0.7225849628448486f, -0.4239010512828827f, 1.3261003494262695f, -1.067254662513733f, 0.6243192553520203f, -2.4506609439849854f, 1.3008989095687866f, -1.3570464849472046f, 0.4011816382408142f, -0.4519657790660858f, -0.5720081329345703f, 1.7311065196990967f, 0.40453165769577026f, 0.5910705924034119f, 0.5206778049468994f, 0.26302585005760193f, 1.462548851966858f, 0.806793212890625f, 2.1060609817504883f, 0.1250244528055191f, -1.0741941928863525f, -0.04460187256336212f, -0.7740861177444458f, 1.4476147890090942f, -1.8094489574432373f, 0.40884074568748474f, -0.13771961629390717f, -0.48222219944000244f, -0.9596375823020935f, 0.88399738073349f, -1.1032230854034424f, 0.4613407552242279f, -0.6266279220581055f, -0.059072766453027725f, 1.432356357574463f, -0.3214779198169708f, -0.32065340876579285f, -1.381635308265686f, -0.36659157276153564f, -1.3767681121826172f, -0.6575090289115906f, -1.0047813653945923f, 0.15763555467128754f, -2.729581594467163f, -0.12834908068180084f, -0.4433963894844055f, -1.7998695373535156f, -0.4873202443122864f, -0.360196590423584f, -0.2588382959365845f, 1.3558588027954102f, -0.6274254322052002f, 0.47283434867858887f, -0.6854686141014099f, 1.4168455600738525f, 0.8578910827636719f, 0.8829494118690491f, -0.8793548941612244f, -0.8100994825363159f, 0.6788665652275085f, 0.33716168999671936f, -0.5290454030036926f, 0.9324967265129089f, 0.6762214303016663f, 0.14378660917282104f, 2.18232798576355f, 0.037647929042577744f, 0.21775013208389282f, 2.395190954208374f, 0.4023849368095398f, 1.6874338388442993f, -2.743908405303955f, -1.9040486812591553f, 0.03124547004699707f, 0.3133164942264557f, -1.2002359628677368f, -0.22739538550376892f, -0.23840101063251495f, -0.682720422744751f, 0.46212872862815857f, -0.9165003299713135f, -1.665152907371521f, 0.7060098052024841f, 0.9033956527709961f, -0.3824882507324219f, 0.1978939324617386f, -0.032255567610263824f, -0.9255485534667969f, 1.1699923276901245f, -1.751257061958313f, 0.9016333222389221f, -1.3840047121047974f, -0.8077333569526672f, 1.2065165042877197f, 0.1894015073776245f, -0.6532776355743408f, -0.2359209805727005f, 0.6057451367378235f, -0.3558703064918518f, 1.4644410610198975f, -0.8236719965934753f, -0.3576895296573639f, -0.9380688667297363f, 1.4365062713623047f, 1.5855119228363037f, 0.39503413438796997f, -0.09881415218114853f, -1.8138774633407593f, -0.6933608055114746f, 1.3770830631256104f, 1.0369852781295776f, 0.522045910358429f, -2.008735179901123f, -0.7871567606925964f, 0.07242458313703537f, -1.9583851099014282f, -0.1682676076889038f, -0.09407275170087814f, -1.5021157264709473f, 0.9083237051963806f, -0.627318799495697f, -1.1001535654067993f, 0.8703535795211792f, 0.9143402576446533f, -0.9333758354187012f, -1.9243427515029907f, 0.5441842079162598f, 1.674156904220581f, -0.1457550972700119f, -0.663499116897583f, 1.2121490240097046f, 0.9221742749214172f, 0.5710628032684326f, -0.7049512267112732f, 2.053586959838867f, 0.9570578932762146f, -1.0222344398498535f, -0.5004768371582031f, -0.8111429214477539f, 2.0269458293914795f, -2.568801164627075f, 1.0234973430633545f, -0.5307855606079102f, 2.509202480316162f, 0.9496951103210449f, 1.4498697519302368f, 1.9099774360656738f, -0.41487377882003784f, -2.154376745223999f, -1.1843920946121216f, -0.13271500170230865f, -0.6129738092422485f, 1.0654757022857666f, 0.589937686920166f, -1.330565333366394f, 1.276455044746399f, -1.1509960889816284f, 1.4542961120605469f, 0.21978320181369781f, -0.26515746116638184f, 0.7749262452125549f, 0.12985540926456451f, 1.0439496040344238f, -1.7788974046707153f, 0.3559875190258026f, -0.05403977632522583f, 0.07127834856510162f, -0.8058504462242126f, -0.35074129700660706f, -0.17826782166957855f, -0.2625838816165924f, 0.7513576745986938f, 0.18278615176677704f, -0.9977543950080872f, 0.45807182788848877f, 0.7924372553825378f, -0.9643475413322449f, 0.46591266989707947f, 1.3539252281188965f, -1.459295630455017f, 0.37469375133514404f, 0.7460726499557495f, 0.26590028405189514f, -1.0854178667068481f, -1.5084643363952637f, 1.1576530933380127f, -0.21968594193458557f, 0.4215473532676697f, -0.32665783166885376f, 2.3412764072418213f, -0.5351880192756653f, 1.0548261404037476f, -0.4031357765197754f, 0.6454427242279053f, -0.6314883828163147f, -1.2809436321258545f, 0.009232764132320881f, 1.2330046892166138f, 0.6238497495651245f, 1.5213422775268555f, -0.4976206421852112f, -1.0449823141098022f, -0.27992430329322815f, 1.2101659774780273f, -0.9885204434394836f, 0.350207656621933f, -0.2644238770008087f, -1.0631657838821411f, -0.7474985122680664f, 0.3266002833843231f, -0.49929699301719666f, -1.4715039730072021f, 0.5023646950721741f, 0.4279913902282715f, 1.0040385723114014f, 1.708235263824463f, -0.2667044997215271f, 0.18906556069850922f, 1.0375696420669556f, 0.7793638706207275f, -0.6498270630836487f, -0.48095330595970154f, 0.13058727979660034f, 0.8354679346084595f, 1.6123450994491577f, -0.4939593970775604f, 0.30456098914146423f, 2.1306238174438477f, 0.026654772460460663f, 0.7856671810150146f, 0.8128212690353394f, 0.04477962106466293f, -1.7439521551132202f, -1.1074808835983276f, -0.6043308973312378f, 0.0144581263884902f, -0.2186170518398285f, -1.0052590370178223f, 1.2696419954299927f, 1.212203025817871f, -0.3618496060371399f, 1.1163915395736694f, 0.4140532910823822f, 0.12140734493732452f, 0.836719274520874f, -0.6638585925102234f, -2.0399112701416016f, -0.6206883192062378f, -0.9584940075874329f, -0.45995032787323f, 0.06401844322681427f, -0.4596543312072754f, -0.5983195900917053f, 0.844002366065979f, -0.6386957168579102f, -1.218558430671692f, -0.1842191219329834f, -1.7404849529266357f, -0.43988513946533203f, 0.978778600692749f, 1.003504991531372f, 1.839215874671936f, -0.7032709121704102f, -2.1381378173828125f, 0.3657694160938263f, -0.01325446367263794f, 1.1153861284255981f, -0.4465695023536682f, 1.315346598625183f, 1.038090705871582f, 2.2033896446228027f, -0.907842755317688f, 2.0937135219573975f, -0.9239965677261353f, -0.553339421749115f, 0.09259668737649918f, 1.0563279390335083f, 0.8357012271881104f, 0.39962950348854065f, -0.5224289298057556f, 1.8076858520507812f, 0.973744809627533f, 1.072774887084961f, -1.0264922380447388f, -1.1302908658981323f, -0.2908953130245209f, -1.6867245435714722f, 1.5600812435150146f, 1.8405765295028687f, -0.7730143070220947f, -0.2413354218006134f, 2.3902435302734375f, -0.5483986139297485f, 0.643738329410553f, 0.7495583891868591f, -1.2053273916244507f, 1.4264872074127197f, -1.1080999374389648f, -0.1472785621881485f, -2.2464029788970947f, 0.40276598930358887f, 0.1035822331905365f, 0.3125080168247223f, 0.19359846413135529f, -0.40304437279701233f, 0.7284680604934692f, 1.4359182119369507f, -1.437017560005188f, 0.5239248275756836f, -0.26981568336486816f, 0.017309220507740974f, 1.2778736352920532f, -0.30529654026031494f, 0.7788206934928894f, -0.08426624536514282f, 0.8460555672645569f, 1.1856203079223633f, -0.4465353190898895f, 0.8045015335083008f, 0.830817461013794f, -0.4116531312465668f, -0.19528378546237946f, 1.0318245887756348f, -0.6419956088066101f, -1.068763256072998f, -0.038084980100393295f, -0.8322976231575012f, 0.8178791999816895f, 0.14809109270572662f, -1.4732903242111206f, -0.10355143249034882f, -0.5564566254615784f, 0.3039886951446533f, 0.9035240411758423f, 0.5980544090270996f, -1.4176050424575806f, -0.0560019314289093f, -1.3687875270843506f, -0.5567623376846313f, -0.36739063262939453f, 1.5230790376663208f, 1.4708421230316162f, -0.4366856515407562f, -1.1750401258468628f, 0.24909113347530365f, 1.8527616262435913f, -1.680544137954712f, 0.659866988658905f, -0.5326191782951355f, 0.5515633225440979f, 1.6462424993515015f, 0.2259528934955597f, 0.27357611060142517f, -0.9556253552436829f, 0.43514177203178406f, -1.1005570888519287f, 0.9130839705467224f, -0.45959728956222534f, 2.1901278495788574f, -0.682377278804779f, -0.22877585887908936f, 0.7370786666870117f, -1.690212607383728f, 0.30514323711395264f, -0.0909213200211525f, -1.1586445569992065f, -0.7481539845466614f, 1.1121810674667358f, -0.4421563148498535f, 1.0719655752182007f, 0.17691805958747864f, -1.2427209615707397f, 0.10128959268331528f, 0.19372011721134186f, 0.24058400094509125f, 0.9057638049125671f, 0.11222076416015625f, -0.059362635016441345f, -2.4078524112701416f, -1.2304182052612305f, -0.026510445401072502f, 0.8861871957778931f, -0.7287083268165588f, 1.380319356918335f, 0.6684188842773438f, -1.7624090909957886f, 0.6292073726654053f, -0.7828742265701294f, -0.08022477477788925f, 0.9383800625801086f, -0.4210285246372223f, -0.49148616194725037f, -0.9306524395942688f, -1.267333745956421f, -0.6902195811271667f, 0.23923389613628387f, 1.4039400815963745f, -2.0282845497131348f, -1.0275602340698242f, 0.9761411547660828f, -0.06743653863668442f, 0.06586836278438568f, -0.7352823615074158f, 0.19450226426124573f, -0.7134578824043274f, 0.16818134486675262f, -1.1888638734817505f, -0.37875133752822876f, -0.321153849363327f, 1.3544011116027832f, -0.423284113407135f, -0.9974878430366516f, -0.9902269244194031f, -0.1406998336315155f, 0.8166614174842834f, 1.0789191722869873f, 1.7252999544143677f, 1.1660224199295044f, 0.5868375301361084f, -0.11877616494894028f, -0.34278497099876404f, -0.08240477740764618f, 0.8542949557304382f, -0.9149259924888611f, 0.5388334393501282f, -1.688096284866333f, -1.05665123462677f, -0.33759868144989014f, -0.2788163125514984f, -0.3783758282661438f, -0.2296314239501953f, -1.1831741333007812f, -1.034787893295288f, -0.8083778023719788f, -0.4777121841907501f, 0.8160690665245056f, 0.8076881766319275f, 0.5698519945144653f, -0.4374706447124481f, -0.7118549942970276f, 2.338996648788452f, -0.0368775799870491f, -0.1774366796016693f, 1.5478016138076782f, 1.0848923921585083f, -0.38529422879219055f, -0.5037281513214111f, 0.9991070628166199f, -0.715931236743927f, 0.28394055366516113f, 0.5939877033233643f, 0.31532949209213257f, 0.27048054337501526f, 0.9632524251937866f, -1.2189302444458008f, -0.06929556280374527f, 0.061022937297821045f, 0.822035551071167f, 0.38992002606391907f, -0.49170032143592834f, -1.689696192741394f, 0.05627693980932236f, -0.2405131459236145f, 0.9160073399543762f, 0.7348390221595764f, -0.6902630925178528f, -0.9186923503875732f, -0.6866548657417297f, -0.10339432954788208f, -0.32694968581199646f, 1.2450132369995117f, 0.6069359183311462f, -0.0937698483467102f, -0.004049783106893301f, 2.6462395191192627f, 1.7450222969055176f, 0.2888743579387665f, 1.1111222505569458f, 1.1171294450759888f, -0.7966887354850769f, 1.0229419469833374f, -1.0742470026016235f, -1.4498995542526245f, -0.5231860280036926f, 0.3438241481781006f, 0.20375612378120422f, -1.4713748693466187f, 0.19777953624725342f, -0.3222813904285431f, 0.5210180282592773f, 1.036444902420044f, 0.2102498859167099f, -1.1502959728240967f, -0.25832024216651917f, 0.2814403772354126f, -0.5159256458282471f, 1.522001028060913f, -0.4228188693523407f, 0.5675636529922485f, 0.47951194643974304f, -0.9628922939300537f, 0.16651970148086548f, 0.17922191321849823f, -0.6149380207061768f, 0.884855329990387f, -0.7095723152160645f, 1.5105708837509155f, 0.9143079519271851f, -1.2950124740600586f, 0.9506115317344666f, -1.3415539264678955f, 0.9584245681762695f, -1.834415316581726f, -0.5945284366607666f, -1.6709949970245361f, -1.3442386388778687f, -0.7921901941299438f, 0.1253465712070465f, -0.9972168207168579f, 0.9316890239715576f, -1.098996639251709f, -0.8827964067459106f, -0.03510960936546326f, 0.33106282353401184f, -1.535096526145935f, 1.5791680812835693f, 1.562730073928833f, 1.0339767932891846f, -0.644834041595459f, -0.7956817150115967f, 0.4747835099697113f, 0.6450149416923523f, -2.4694929122924805f, -1.2703917026519775f, 1.4028544425964355f, 0.39471593499183655f, -0.2914227247238159f, -0.04415971040725708f, 0.6573874354362488f, -1.8288997411727905f, 0.515746533870697f, -2.1813249588012695f, -0.7443293333053589f, 0.4601408839225769f, 1.2225193977355957f, -2.047572374343872f, 1.09447181224823f, 0.42030036449432373f, -0.6366481781005859f, -2.1695642471313477f, 1.1429638862609863f, 0.8955739736557007f, 1.6797635555267334f, -0.6834523677825928f, 0.8514192700386047f, -0.8422428369522095f, -1.0746608972549438f, -0.9010616540908813f, -0.6362974643707275f, -1.087014079093933f, -0.028106562793254852f, -0.1531383991241455f, 1.0988678932189941f, -0.03311381861567497f, -0.892038881778717f, -1.3073756694793701f, 0.14662222564220428f, 0.8670119643211365f, 3.2090775966644287f, -1.3412030935287476f, 0.3424357771873474f, 0.19632267951965332f, -0.1660011112689972f, -0.3333166837692261f, 1.0084987878799438f, 0.9129180312156677f, 0.5135545134544373f, -0.8764104843139648f, -1.4057948589324951f, -0.6906246542930603f, 0.3960992097854614f, -0.2924385964870453f, 0.3751489818096161f, -1.1075292825698853f, 0.37509971857070923f, -1.4790199995040894f, 0.5692160725593567f, -1.146249532699585f, -1.904903769493103f, 0.9906712174415588f, -0.9967307448387146f, -2.3042960166931152f, -1.8701173067092896f, 1.6571006774902344f, -0.5442659258842468f, 0.3686337471008301f, -0.17834435403347015f, -0.7094868421554565f, 0.24787208437919617f, 0.7445037364959717f, 1.5205289125442505f, -1.3338558673858643f, -2.0718472003936768f, -1.8923321962356567f, 2.419602632522583f, -0.24423770606517792f, -0.2663561701774597f, -0.6755290031433105f, -1.124692440032959f, 1.8602604866027832f, 0.6256695985794067f, 0.28849464654922485f, 0.8962987065315247f, 0.8840014338493347f, 0.4107360541820526f, -1.6067783832550049f, 1.4667541980743408f, -1.643362045288086f, -0.7528584599494934f, -0.8185959458351135f, 0.02291426621377468f, -0.7729488015174866f, -0.039141006767749786f, 0.30586403608322144f, -0.2691587805747986f, -0.21813122928142548f, 0.3616786003112793f, -1.017099380493164f, 0.7204318046569824f, -0.10523297637701035f, 1.8706086874008179f, 0.8992230892181396f, -0.23518913984298706f, -0.6936088800430298f, 0.38842713832855225f, -0.9864745140075684f, 0.24743980169296265f, 1.3209456205368042f, 0.5116807222366333f, 0.20717298984527588f, 0.423617959022522f, 0.8553113341331482f, -0.7523386478424072f, -0.7315530180931091f, -0.34988582134246826f, 0.14683414995670319f, 0.337516725063324f, -0.5604003667831421f, -0.3334929347038269f, 0.46565955877304077f, -0.831680178642273f, -0.8464822769165039f, -0.38647857308387756f, -0.1513155996799469f, 0.8186390399932861f, -0.25432348251342773f, -0.47535333037376404f, -0.780207633972168f, 0.778557300567627f, 0.0986286848783493f, -0.754118800163269f, 0.9054679274559021f, -1.1227487325668335f, 0.2190994918346405f, -0.09212681651115417f, 1.8318346738815308f, -0.5303875803947449f, -0.8198762536048889f, -0.8232145309448242f, 0.5506033301353455f, -1.1416276693344116f, 1.8735969066619873f, 1.3694992065429688f, -1.257182240486145f, -1.6220338344573975f, -0.5663302540779114f, 0.6658836007118225f, 0.25084203481674194f, 0.7626033425331116f, -0.9630534052848816f, -0.5952125191688538f, -0.5548334121704102f, 1.2073006629943848f, -0.726753830909729f, -0.25774624943733215f, 0.7335841059684753f, -0.6472665667533875f, 0.4566407799720764f, -0.506052553653717f, -0.07941094785928726f, 0.226389080286026f, 1.392283320426941f, -0.1649594008922577f, -0.8620851635932922f, 1.041149377822876f, -0.18962401151657104f, 2.0505990982055664f, -2.2949159145355225f, -0.8339797258377075f, 2.1567959785461426f, 1.0796829462051392f, -0.815654456615448f, 0.7114473581314087f, -0.24212278425693512f, 0.9910399317741394f, 0.551307737827301f, 0.6295360326766968f, 0.1759207546710968f, 1.6339221000671387f, 1.2599411010742188f, -0.6335592269897461f, -0.8656108379364014f, 0.2853967845439911f, -1.4132448434829712f, 1.7862261533737183f, -0.18690960109233856f, 0.45671215653419495f, 0.5091333985328674f, 0.12447594106197357f, -0.7764351963996887f, -0.9075336456298828f, 1.6591986417770386f, -0.9774589538574219f, -1.437660813331604f, -1.0792182683944702f, -1.0316091775894165f, 0.4912571609020233f, -0.9918735027313232f, 0.37739869952201843f, -0.09308940172195435f, -0.3923634886741638f, -1.9273265600204468f, -0.9188506603240967f, 1.127413034439087f, 1.4187533855438232f, -0.30123433470726013f, -1.6901737451553345f, 0.9281692504882812f, -1.123509168624878f, -1.0524919033050537f, -0.4467827379703522f, 0.523188054561615f, -0.3387533724308014f, -1.7577482461929321f, -0.6524283289909363f, -0.24884779751300812f, -1.2434697151184082f, 1.6212447881698608f, 0.15123429894447327f, 1.4580625295639038f, 0.8593719601631165f, 0.4116728901863098f, -0.3829646706581116f, 0.20710650086402893f, -0.845027506351471f, 0.2616454064846039f, -0.790607750415802f, -0.35345759987831116f, -0.4279586672782898f, 1.68895423412323f, 1.2128044366836548f, 0.9204495549201965f, 0.047616202384233475f, 0.7502005696296692f, -1.6819013357162476f, -0.036870624870061874f, -0.05854296684265137f, 0.6686491370201111f, -0.4365732967853546f, -2.3380324840545654f, 1.8583958148956299f, -1.7244455814361572f, -0.2750453054904938f, 0.4532167315483093f, 1.2953895330429077f, 0.058747299015522f, -0.8122621774673462f, 1.9493451118469238f, 0.7995834350585938f, -0.010960266925394535f, -0.13873475790023804f, -1.9486329555511475f, -0.7052865028381348f, -0.9286441802978516f, 0.5382292866706848f, -1.931458592414856f, 0.17587143182754517f, 1.2172441482543945f, 0.9002020955085754f, 1.1507850885391235f, 0.027491796761751175f, -1.4959322214126587f, -0.18092377483844757f, 0.905379056930542f, 1.720672845840454f, 0.3225809633731842f, 0.5499839782714844f, -0.6421884298324585f, 0.22866190969944f, -0.5370370149612427f, 0.5335260629653931f, -1.0596036911010742f, -2.2213568687438965f, 0.7067712545394897f, -0.21388542652130127f, -0.4895268678665161f, 0.5013054013252258f, -0.9222452044487f, 0.15941572189331055f, -1.377753734588623f, 1.15412437915802f, -1.092998743057251f, -0.8713886141777039f, 0.22347310185432434f, 0.2528358995914459f, 0.21688957512378693f, 0.4699849486351013f, 0.3267545700073242f, -0.09539508819580078f, -0.18269990384578705f, 0.3382812440395355f, -0.6900221109390259f, 0.047023314982652664f, 0.5487117171287537f, -1.9950600862503052f, -0.1581975668668747f, -0.1996922492980957f, 0.46645262837409973f, 0.5603392124176025f, 0.9167394638061523f, -1.983751893043518f, -0.015853818506002426f, 0.033756233751773834f, -1.4661213159561157f, -0.32681238651275635f, -0.5929558873176575f, 0.8652521371841431f, 0.4256573021411896f, -0.2335444986820221f, -0.7175306677818298f, -2.244840621948242f, 1.9783128499984741f, 0.3803824186325073f, 0.04183865338563919f, 0.284446656703949f, -2.161471128463745f, -0.5391823053359985f, 0.3041086494922638f, -0.3503345549106598f, 1.171104907989502f, -0.8915714621543884f, 1.7047383785247803f, -1.7130810022354126f, 1.0918960571289062f, 1.9015947580337524f, -0.2993949353694916f, 0.2431759536266327f, -0.06963914632797241f, -1.9186826944351196f, -1.1858599185943604f, 1.3959892988204956f, 0.31790241599082947f, 1.0485270023345947f, -0.8889532089233398f, -0.1494847685098648f, 0.011521455831825733f, -0.3648742437362671f, -0.14563268423080444f, 1.1415576934814453f, 2.0192317962646484f, -1.1988294124603271f, -0.08969400823116302f, 1.01350998878479f, -0.148199200630188f, -0.20676261186599731f, -0.4151966869831085f, -0.5613080859184265f, -1.5219647884368896f, -0.6867799758911133f, -1.207437515258789f, 0.27414625883102417f, 1.5931971073150635f, 0.8239222168922424f, 0.17494884133338928f, 0.37811675667762756f, -0.5250972509384155f, 1.1867752075195312f, 0.44894057512283325f, 0.6020882725715637f, 0.27788910269737244f, 0.3914108872413635f, -0.669601321220398f, -0.08302459120750427f, 0.06917848438024521f, -0.263455331325531f, 0.7421362996101379f, 0.2899683713912964f, -0.4380805194377899f, -1.3862485885620117f, -0.7712535858154297f, -0.3168620467185974f, -1.6128871440887451f, -0.9478936195373535f, -0.37378421425819397f, -0.8723973631858826f, 0.43149101734161377f, -0.7728766202926636f, -0.8030177354812622f, 0.46446937322616577f, -0.17089581489562988f, 2.706796407699585f, 0.6624157428741455f, -0.654021143913269f, 0.7278003692626953f, 0.09257330745458603f, -0.1797974407672882f, 0.7003864645957947f, -1.250577688217163f, 0.9090378284454346f, -0.15779435634613037f, -0.43905171751976013f, -0.16366083920001984f, -1.2389750480651855f, -1.0557494163513184f, 1.2461082935333252f, -0.7245869040489197f, -1.044531226158142f, 0.5626670122146606f, 0.17291614413261414f, 1.9904685020446777f, -1.181625247001648f, 0.30392396450042725f, 2.225896120071411f, 0.30583375692367554f, -0.6297805309295654f, 0.7762312889099121f, -0.5106104016304016f, -0.5062925219535828f, 0.7492969632148743f, 1.0681610107421875f, -2.636631488800049f, 0.6954406499862671f, -0.32303762435913086f, 0.5969774723052979f, -0.6247759461402893f, 1.647339940071106f, -1.7078118324279785f, 0.09625459462404251f, -0.28645211458206177f, 1.0167992115020752f, -0.6579540371894836f, -0.3999125063419342f, -1.7973577976226807f, -0.9080927968025208f, 1.1592087745666504f, -1.467360258102417f, 1.7191014289855957f, -0.3673190176486969f, 2.3575830459594727f, -0.459722101688385f, -0.058640673756599426f, -1.2418878078460693f, -0.023634955286979675f, -2.0428922176361084f, 0.19053655862808228f, 0.5756992697715759f, -0.5177616477012634f, 0.9197074770927429f, 0.9861821532249451f, 0.9352383017539978f, 0.30303436517715454f, -0.5343928933143616f, -0.2886335253715515f, 0.9647671580314636f, 1.7285969257354736f, -0.8043005466461182f, -0.28731220960617065f, -1.2584627866744995f, 0.08387812972068787f, 0.9973886609077454f, -0.7661278247833252f, -0.3684079647064209f, -0.16042770445346832f, 0.731488823890686f, -0.4989534318447113f, 0.5183284878730774f, 0.6223585605621338f, 0.017574317753314972f, 1.3758805990219116f, -0.024098770692944527f, -0.11441737413406372f, -0.2840602993965149f, -0.10638472437858582f, 0.3422453701496124f, -0.21973253786563873f, -0.09440521895885468f, -0.514839768409729f, 0.5137941241264343f, 0.3945278227329254f, 0.5612776279449463f, -0.006535662803798914f, 0.8489260077476501f, 0.4735972583293915f, 0.5418957471847534f, 0.0022529142443090677f, -0.5435486435890198f, 0.18075302243232727f, -0.7696079015731812f, -1.0346392393112183f, 0.7314628958702087f, 1.6726516485214233f, 0.35546284914016724f, 0.30347952246665955f, 0.04806268960237503f, 0.5032230019569397f, -0.025565603747963905f, -1.8534711599349976f, 1.0129450559616089f, -0.07218152284622192f, 0.03116939589381218f, -0.755984902381897f, -0.7956190705299377f, -0.6868169903755188f, 1.9091075658798218f, -0.4942598342895508f, -0.06308790296316147f, 0.9503589868545532f, 0.6692175269126892f, 0.12500713765621185f, 0.5085462331771851f, -1.0646655559539795f, -0.4647725224494934f, -0.9152674674987793f, 0.7144214510917664f, 1.472544550895691f, -1.7723859548568726f, 0.0483609139919281f, 0.20343580842018127f, -0.6890958547592163f, -0.5412123203277588f, 2.0465071201324463f, -0.5884720087051392f, -0.8352398872375488f, 0.0502433180809021f, 0.4375612735748291f, 1.3833050727844238f, -1.3894257545471191f, 0.3897654116153717f, -1.5383917093276978f, 1.8197290897369385f, -0.5204964280128479f, 1.4799479246139526f, 0.4248008131980896f, -0.30355918407440186f, 0.07046826928853989f, -1.5127924680709839f, 1.3581115007400513f, 0.32402706146240234f, -1.0680902004241943f, 0.08959023654460907f, 3.1311562061309814f, -0.8544708490371704f, -0.8357695937156677f, 1.2985942363739014f, -0.7110929489135742f, -0.9653112292289734f, -1.340206503868103f, 0.03399703651666641f, -0.4458200931549072f, 0.151767760515213f, 1.6231015920639038f, -0.429679274559021f, -0.16159531474113464f, -1.0276970863342285f, -0.9918712973594666f, 0.29410406947135925f, -0.3595946729183197f, -0.13289135694503784f, -0.09766664355993271f, -1.9935065507888794f, 0.11608057469129562f, 0.9187806844711304f, 0.12344544380903244f, 2.8273682594299316f, 0.47407886385917664f, -1.9149128198623657f, -1.5384353399276733f, -0.6529980897903442f, 0.16414928436279297f, -0.19700200855731964f, 1.038715124130249f, -0.3908529281616211f, 1.0207839012145996f, -1.2268577814102173f, -2.0579991340637207f, 0.5987722277641296f, -0.13595744967460632f, 0.24731674790382385f, -2.150991439819336f, 1.2150318622589111f, -0.33008328080177307f, -0.5156939625740051f, 0.022917015478014946f, -0.6746217608451843f, 1.309282898902893f, -1.6657313108444214f, 0.32850465178489685f, -0.8491258025169373f, 0.9747982025146484f, -0.37322181463241577f, 0.6090918779373169f, -0.7589855194091797f, -0.6492360830307007f, -1.2674946784973145f, -0.382905513048172f, 0.8315308690071106f, 1.8437397480010986f, 0.021685972809791565f, 0.28762421011924744f, 0.39912644028663635f, 2.4587013721466064f, 0.4567311406135559f, 1.106606125831604f, 0.9169553518295288f, -0.23127765953540802f, -1.498667597770691f, 0.6427391171455383f, 0.18222902715206146f, -1.0757185220718384f, 1.345549464225769f, 0.32771772146224976f, 1.3408823013305664f, 1.2158557176589966f, 0.95885169506073f, 0.5136737823486328f, 0.5763065814971924f, 0.0778578370809555f, -1.061161994934082f, 2.042306900024414f, 0.6509259343147278f, -1.0072094202041626f, 0.35781362652778625f, -1.0799492597579956f, 0.04710597172379494f, 2.201401472091675f, 0.2908516526222229f, -0.08251824229955673f, 1.5195056200027466f, -0.3584267199039459f, -0.6102548837661743f, -1.5060930252075195f, -0.7732543349266052f, 1.0847911834716797f, 1.4872429370880127f, 0.6041548848152161f, -1.1257244348526f, 0.3761575222015381f, 1.959293246269226f, -0.7958163619041443f, 0.4539487659931183f, -2.5069937705993652f, -1.361000418663025f, 1.183761477470398f, -1.1673825979232788f, -0.25077250599861145f, 0.33264845609664917f, 1.1230510473251343f, 1.8761743307113647f, 0.273639976978302f, 0.2801147401332855f, 0.7281787991523743f, 0.02403142862021923f, 2.3937904834747314f, -1.3129955530166626f, -0.48624828457832336f, 0.4012764096260071f, 1.0767757892608643f, 1.1279785633087158f, 0.02586616761982441f, -0.14947263896465302f, 0.39506977796554565f, 1.9742354154586792f, -0.4404173791408539f, 2.3815414905548096f, -0.34589144587516785f, 2.694831609725952f, -0.8544124364852905f, -0.27781322598457336f, 0.7772916555404663f, -0.43923330307006836f, 1.1729096174240112f, 1.397361159324646f, -0.41255858540534973f, -0.4339471161365509f, 0.9574882388114929f, 1.254128336906433f, -1.0384563207626343f, 0.5251074433326721f, -0.8383287787437439f, 1.7227904796600342f, -1.1731188297271729f, 0.7272883057594299f, -1.7194626331329346f, 2.383774995803833f, -1.1649107933044434f, -0.20503655076026917f, -0.5375787019729614f, 1.2967820167541504f, -1.6138769388198853f, 0.7535606026649475f, -0.38888975977897644f, 2.5921998023986816f, 0.08543383330106735f, -2.0308878421783447f, 0.6461803913116455f, 1.217452883720398f, 0.03121299482882023f, 1.082465648651123f, -1.8998514413833618f, -2.7074427604675293f, -0.7867730259895325f, -0.6195639967918396f, 1.8281400203704834f, 0.4877544343471527f, 0.1366790533065796f, -0.5910199284553528f, 2.3618080615997314f, 1.2076122760772705f, -0.5487976670265198f, 1.5311497449874878f, 0.44362834095954895f, -2.4579336643218994f, 0.6496618390083313f, 0.02449978142976761f, 0.25982725620269775f, 0.7348682284355164f, -0.006029692944139242f, -0.7884382605552673f, 1.1371253728866577f, -1.7366209030151367f, 0.9614791870117188f, -1.3104760646820068f, -1.4218922853469849f, -0.3201002776622772f, 0.011662798002362251f, 0.7528783082962036f, -0.8849666118621826f, 0.8982152342796326f, -1.6744109392166138f, -1.1447242498397827f, 0.22346419095993042f, -0.9369489550590515f, -0.17544429004192352f, 0.8354768753051758f, -1.2516489028930664f, 0.26434803009033203f, 1.2949540615081787f, -0.5020677447319031f, 0.45930108428001404f, -1.3068112134933472f, 0.9302626848220825f, 1.0709624290466309f, 1.3384746313095093f, 0.27829068899154663f, -1.328766107559204f, -0.2891634404659271f, -2.518200635910034f, -2.533468723297119f, -0.022988365963101387f, -0.5563327670097351f, 2.0752601623535156f, -0.8403553366661072f, 0.9517558813095093f, -2.1648738384246826f, 0.9794597625732422f, 0.38047143816947937f, -1.9277540445327759f, 0.9197121858596802f, -1.6067678928375244f, -1.3559657335281372f, -0.25936615467071533f, 0.0861901044845581f, -0.37659966945648193f, 0.9756653904914856f, -0.8411983847618103f, -0.9621247053146362f, -0.5808306932449341f, 1.1524169445037842f, -0.5585207939147949f, -0.13208438456058502f, -1.0929938554763794f, -2.4860122203826904f, -0.038849566131830215f, 0.8021038770675659f, -0.18547378480434418f, 0.534229576587677f, -0.49810588359832764f, 0.8963930010795593f, -1.2691730260849f, 0.042280785739421844f, 0.9680984020233154f, -1.7608284950256348f, 1.0980932712554932f, 0.8579375743865967f, -0.9562888145446777f, -0.7236151099205017f, 0.2793406844139099f, -0.8662601113319397f, 0.8459339737892151f, 0.6116387844085693f, -0.8089803457260132f, -0.20030127465724945f, -2.4035587310791016f, 1.5633418560028076f, -0.6294617652893066f, 0.8042920231819153f, 0.6676139235496521f, -0.5181459784507751f, 0.9539962410926819f, 0.7256157398223877f, -0.5767995715141296f, -0.01296310406178236f, 1.34257972240448f, 0.8595607280731201f, 0.24403470754623413f, 1.9769524335861206f, 0.5327519178390503f, -0.8788067102432251f, -0.2609003186225891f, -0.1277228742837906f, -0.38701093196868896f, -0.10875839740037918f, 0.7135027647018433f, 0.037005677819252014f, 0.7367677688598633f, -1.073968529701233f, -0.8155962824821472f, 0.3688916862010956f, 0.5232675075531006f, -1.1358050107955933f, 2.01452898979187f, -0.5778390765190125f, -0.2187574803829193f, 0.35567042231559753f, 0.7510744333267212f, 0.2171924114227295f, 0.4019394814968109f, 0.9250243306159973f, -0.24931922554969788f, -0.3473069369792938f, 0.8122217059135437f, -0.6078834533691406f, -0.8096639513969421f, -0.4163765013217926f, -0.9129456877708435f, -1.0231975317001343f, 0.15973883867263794f, 1.5596561431884766f, 0.2012278288602829f, 0.9135115742683411f, -0.7519399523735046f, 2.7958858013153076f, 0.41675102710723877f, 0.0033115341793745756f, -1.398554801940918f, 1.5899420976638794f, -0.7806249856948853f, -0.7495994567871094f, 0.2814400792121887f, -0.08736804872751236f, -1.1191902160644531f, -0.4140253961086273f, 0.2933676540851593f, -0.016588633880019188f, 0.4729222357273102f, 1.0479241609573364f, 0.32106152176856995f, 1.162279486656189f, 1.5198776721954346f, 1.134521245956421f, 0.013782661408185959f, 1.5933806896209717f, 0.11106725037097931f, 0.5933053493499756f, 0.0023419519420713186f, 0.9275448322296143f, 0.1871725618839264f, 1.0549991130828857f, -0.04648401960730553f, 1.6230204105377197f, 0.9345911145210266f, 0.6804553866386414f, 2.2367258071899414f, -0.6858885884284973f, -1.308877944946289f, -1.208367109298706f, -1.4461109638214111f, -0.6192750930786133f, -0.40478718280792236f, 0.45292869210243225f, 0.40945965051651f, 0.9656304121017456f, 0.4625106155872345f, -1.4208135604858398f, 0.2507438659667969f, -0.5702037215232849f, -0.05394743010401726f, -0.7306843400001526f, -1.0346637964248657f, 0.7631427049636841f, 0.9210829138755798f, -0.6593214273452759f, 0.24809309840202332f, 0.7311660647392273f, 2.3288936614990234f, -1.06203031539917f, -0.5385692119598389f, 0.4127616584300995f, 0.3390391767024994f, 1.7734715938568115f, 1.655669927597046f, -0.08070362359285355f, -0.303115576505661f, 0.3240552544593811f, 1.0401465892791748f, 0.5915576815605164f, -1.4803826808929443f, -0.12040789425373077f, 0.16437974572181702f, 0.7806977033615112f, 1.228264331817627f, 0.4453698992729187f, -0.3023916184902191f, -0.7796450257301331f, 1.15239417552948f, -0.4312492907047272f, 0.4804547131061554f, -0.0037577631883323193f, -0.23722761869430542f, 0.1753881573677063f, -0.3416663408279419f, -0.4271427094936371f, 1.1991220712661743f, 0.8689031600952148f, -0.6073640584945679f, 0.11616495996713638f, 1.9142365455627441f, -1.1450425386428833f, 1.60905921459198f, -0.5139697194099426f, -0.3140712380409241f, -0.8430630564689636f, -1.2868314981460571f, 0.5520419478416443f, 2.584942102432251f, 0.4215601086616516f, 2.558887481689453f, -1.2483208179473877f, -0.9826495051383972f, 0.7373914122581482f, 1.4831291437149048f, 0.6203920245170593f, 0.33499792218208313f, -0.5796453952789307f, -0.24162966012954712f, 1.4769864082336426f, -1.1677722930908203f, 0.43188926577568054f, 0.7582098245620728f, 0.26907673478126526f, -0.16138136386871338f, 1.1750508546829224f, 0.5351220369338989f, 1.225508689880371f, 0.9085782766342163f, 0.16776901483535767f, -0.009964917786419392f, -0.9104782342910767f, 0.3974350690841675f, -0.5543625354766846f, 1.281534194946289f, 0.38322561979293823f, 1.3406338691711426f, -0.43185070157051086f, -1.897344708442688f, -2.652250289916992f, -0.43282368779182434f, 0.932184100151062f, 0.4349152445793152f, -0.7768413424491882f, -0.19172848761081696f, 1.1136181354522705f, -1.214506983757019f, 2.1966660022735596f, 0.41227126121520996f, -0.7043505311012268f, 0.09847358614206314f, 1.8926005363464355f, 0.6903649568557739f, 1.4352598190307617f, 1.0864993333816528f, -0.3989373445510864f, -2.0487279891967773f, -1.1373023986816406f, 0.9046337604522705f, 0.5279921293258667f, -0.5005130171775818f, 0.0378837063908577f, -1.0129538774490356f, 0.7843693494796753f, 1.3280545473098755f, -0.5138816833496094f, -0.04832516983151436f, -0.8721303939819336f, -1.3769567012786865f, -1.8487751483917236f, 0.30561649799346924f, -0.42448726296424866f, 0.6269863247871399f, -0.6438542604446411f, 3.0598578453063965f, 0.8688741326332092f, 0.7331935167312622f, 1.7552188634872437f, -0.24250611662864685f, 1.9880743026733398f, -1.1370364427566528f, -2.2499585151672363f, -0.5810695886611938f, 0.8816671967506409f, -2.0325005054473877f, -0.35224223136901855f, 0.1006089523434639f, 1.5255436897277832f, -1.6215003728866577f, 0.5298449397087097f, -1.3718810081481934f, -0.6298099756240845f, -0.65668123960495f, 0.4244276285171509f, -0.40122607350349426f, -0.4890124797821045f, 0.9571828246116638f, -0.8436367511749268f, -0.3680597245693207f, -0.47244831919670105f, -0.6993737816810608f, 0.4523862302303314f, -1.090578317642212f, 0.9411556720733643f, -1.931666612625122f, 0.1285097301006317f, -1.0134315490722656f, 0.7479439973831177f, 0.20112790167331696f, -0.8995627760887146f, -0.2906798720359802f, 0.1047126054763794f, -0.5943686366081238f, -1.727795124053955f, 1.0557186603546143f, -0.5037193298339844f, -1.5121967792510986f, -0.14455023407936096f, -0.7709440588951111f, 1.3850905895233154f, 0.06768468767404556f, -2.205592393875122f, -0.9360390901565552f, 1.6600853204727173f, -0.551665186882019f, -0.3103943169116974f, -0.6468674540519714f, -0.22681531310081482f, -0.3377947509288788f, 0.6926270723342896f, -0.5544835925102234f, -0.10487986356019974f, -0.12251695245504379f, -0.10804381966590881f, -1.4534186124801636f, -0.5115614533424377f, 0.9907002449035645f, 0.7906493544578552f, -0.34170281887054443f, -1.8583146333694458f, 0.6896669268608093f, 0.8880993127822876f, 1.2050648927688599f, -0.5951094031333923f, -0.3993092179298401f, -0.30013880133628845f, -0.014673005789518356f, 0.6156877279281616f, -0.9164139032363892f, -0.06423450261354446f, -0.33351314067840576f, 0.5679208636283875f, 0.5063413977622986f, 1.8924111127853394f, -0.7531763911247253f, 1.0589748620986938f, -0.010384922847151756f, 0.41770482063293457f, 0.16313597559928894f, -0.26724591851234436f, 0.1958005577325821f, 0.8279790282249451f, -0.7302268147468567f, 0.4149245321750641f, 2.0801644325256348f, 0.1493149846792221f, -0.4159306585788727f, 1.2188113927841187f, 0.8605815768241882f, 0.8988009691238403f, -0.4320070743560791f, -0.011584565043449402f, 0.1905190795660019f, 0.45351630449295044f, -0.2722189724445343f, 0.4377744197845459f, 0.6674123406410217f, -0.8114345669746399f, -1.1677125692367554f, 1.4608654975891113f, -1.6388529539108276f, 0.9854339957237244f, 0.01134670153260231f, 0.3140886127948761f, 1.081308126449585f, 0.1636349856853485f, -1.8501884937286377f, -0.6727942824363708f, -1.8205945491790771f, -0.16954289376735687f, -0.974631667137146f, -0.5593003630638123f, -0.2645290195941925f, 0.2448410987854004f, 0.4654839038848877f, 1.1221051216125488f, -1.2755296230316162f, 1.968849539756775f, 1.1931818723678589f, -0.9534568786621094f, 0.6892605423927307f, -0.03748741000890732f, -1.3567429780960083f, -0.930943489074707f, -0.4104326069355011f, 1.6298463344573975f, -0.3964575529098511f, -0.8092941045761108f, -0.8157675862312317f, -0.1790880262851715f, -0.7333499789237976f, 2.0713958740234375f, 0.26833969354629517f, -0.4909171462059021f, 0.0867118388414383f, -0.19213901460170746f, -0.020139964297413826f, 1.3420833349227905f, 0.8771296143531799f, 0.9378947615623474f, 0.023886969313025475f, 1.5275574922561646f, -0.08036366105079651f, -0.9117053151130676f, -1.1944957971572876f, 0.9197462201118469f, 0.28947049379348755f, 0.5755492448806763f, -1.0247280597686768f, 0.19418835639953613f, 1.5901795625686646f, -0.7810332179069519f, 0.18075843155384064f, 0.5682910680770874f, -1.3145383596420288f, -0.037264980375766754f, -1.1747896671295166f, -0.5241218209266663f, 0.8552600145339966f, 0.7151663303375244f, 0.854972243309021f, 0.6970289349555969f, -0.7869400978088379f, -0.22646142542362213f, 0.8945121765136719f, -0.07112658768892288f, 0.19750215113162994f, -0.08805552124977112f, 0.46580448746681213f, 0.745457112789154f, -0.6745474338531494f, -0.8672813177108765f, -2.0061123371124268f, 2.204427480697632f, -0.336713045835495f, -0.07664714008569717f, 0.6008157134056091f, 0.7986276745796204f, -1.3824840784072876f, -0.3528830111026764f, 0.9141533374786377f, 0.5920729637145996f, 0.7283052206039429f, 0.5198004245758057f, 0.40039631724357605f, -0.16163653135299683f, 0.3812867999076843f, 0.2818247973918915f, -0.259854257106781f, -0.17636136710643768f, 1.2899055480957031f, -1.809929609298706f, 0.04499081149697304f, -0.5114196538925171f, 1.6922872066497803f, -0.5404288172721863f, -0.06902796775102615f, -0.900188148021698f, 0.16361606121063232f, -0.5697844624519348f, -0.17199715971946716f, 1.9494951963424683f, -1.5725066661834717f, -0.11967476457357407f, 0.6768812537193298f, -2.550161600112915f, 1.3886760473251343f, 2.071373224258423f, -0.4157007336616516f, 1.7397890090942383f, -1.5081530809402466f, -0.9471526145935059f, 0.4156656265258789f, -0.7409192323684692f, -2.0307180881500244f, 0.7459039092063904f, -0.23346112668514252f, -1.7168304920196533f, -1.5648266077041626f, 1.1608803272247314f, -1.7281652688980103f, 0.6240199208259583f, 0.19563332200050354f, 0.9567276835441589f, 0.9916862845420837f, -0.7535449862480164f, 1.3851630687713623f, 0.8076578974723816f, 0.4444909691810608f, 0.12085723131895065f, 0.794389545917511f, -0.2691969871520996f, 1.6320013999938965f, 1.4558833837509155f, 0.8513883352279663f, 1.305290937423706f, 0.8631715178489685f, -0.5379660725593567f, 0.5391438603401184f, -0.18788452446460724f, -1.6509469747543335f, 0.6196591854095459f, -1.1270289421081543f, -0.08517079800367355f, 1.0406969785690308f, -0.5241701006889343f, -0.5131880044937134f, 0.4295358955860138f, 1.8150509595870972f, -1.150861382484436f, -0.30584630370140076f, 0.8076725602149963f, 0.7956660985946655f, 0.2235465794801712f, -0.05132210999727249f, 0.7380313277244568f, 1.5020791292190552f, 0.8205595016479492f, -0.056694693863391876f, -0.17149198055267334f, 0.9732901453971863f, -0.9835246801376343f, 0.1294081211090088f, -1.1072022914886475f, -0.1295740306377411f, 1.005579948425293f, -1.3748044967651367f, 1.197999119758606f, -2.2044677734375f, -1.1458145380020142f, -1.4537255764007568f, -0.2384955883026123f, -1.6215006113052368f, -0.3137211501598358f, -0.7217759490013123f, -1.6709321737289429f, 0.3768741488456726f, -1.8265146017074585f, 0.48899734020233154f, -0.5580688714981079f, 0.37294480204582214f, -0.6878536939620972f, -1.4576401710510254f, -0.6108043789863586f, -0.6761788725852966f, -1.9144877195358276f, 0.9340372085571289f, -0.4341937303543091f, 1.4776188135147095f, 0.7693286538124084f, 2.331334352493286f, -0.6489352583885193f, 0.4903559982776642f, -0.2804304361343384f, 0.941003680229187f, -0.11338216066360474f, -0.8615462183952332f, 0.13828350603580475f, 0.013346405699849129f, 0.33505237102508545f, -0.24309676885604858f, -0.6505504250526428f, -0.33003056049346924f, 1.3748613595962524f, -1.033202886581421f, 0.05953902378678322f, -0.2076457291841507f, -0.9487320184707642f, -0.2869058847427368f, -0.902455747127533f, -1.5298161506652832f, -1.2186866998672485f, 1.1458920240402222f, 0.20151416957378387f, -0.4259151816368103f, -0.1506912112236023f, -2.106266498565674f, -0.6301612257957458f, 1.2424033880233765f, -0.025852564722299576f, -0.540851891040802f, -0.3480299413204193f, -0.7818117141723633f, 0.5524802803993225f, 0.8857057690620422f, -1.3390332460403442f, -0.40026965737342834f, 0.7468986511230469f, -0.3904872536659241f, 0.7475174069404602f, -0.4455646276473999f, 1.0798193216323853f, 2.6205930709838867f, 0.6864228844642639f, 1.5354714393615723f, -0.1950068324804306f, -1.4005751609802246f, 0.4377126097679138f, -0.05925627797842026f, -0.552335798740387f, -0.7325940728187561f, 0.5038686990737915f, -0.18185997009277344f, -0.7621927261352539f, -0.8477222323417664f, 0.6641071438789368f, 2.3913745880126953f, 0.4669826030731201f, 0.7512773275375366f, -0.3845667541027069f, -0.06630031019449234f, 0.6945987939834595f, -0.7470712065696716f, 0.2076980471611023f, -0.12689319252967834f, -0.5187065005302429f, 0.8009016513824463f, -0.7448427081108093f, 1.9110214710235596f, 0.3227183520793915f, -0.5433307886123657f, -2.0223824977874756f, 0.7815577387809753f, 1.8362489938735962f, -0.47440794110298157f, -0.38593149185180664f, 0.36727604269981384f, 0.344801127910614f, 0.5554131865501404f, 0.35217124223709106f, 0.7673863768577576f, 0.08270980417728424f, 0.3816434144973755f, -1.0023962259292603f, -0.22075708210468292f, -0.6525165438652039f, -2.485137939453125f, 0.614276111125946f, 0.9891844391822815f, -1.6560537815093994f, 0.17567120492458344f, 0.8857656121253967f, 0.022398322820663452f, -0.24084077775478363f, 1.4238216876983643f, 0.8160249590873718f, 0.2655353844165802f, 0.744394838809967f, -0.17409075796604156f, -0.9177995324134827f, -2.1325347423553467f, -0.4131704568862915f, -0.4764634966850281f, 0.5031798481941223f, 0.27043330669403076f, -0.04328599572181702f, 1.035592794418335f, 1.341705560684204f, -0.11890244483947754f, -0.07687754184007645f, 0.5947466492652893f, 0.5723721385002136f, 2.663830518722534f, 0.7388755679130554f, -0.46826601028442383f, -1.59627366065979f, -1.6671663522720337f, 0.3388274610042572f, 0.3750116229057312f, -1.3291982412338257f, 0.5636889338493347f, 0.8051766753196716f, 0.7449150681495667f, -0.1739114224910736f, 1.1078135967254639f, 0.5147720575332642f, 0.8934884667396545f, -1.5113967657089233f, -0.8514725565910339f, 0.3001462519168854f, 0.17312221229076385f, 0.08587908744812012f, 0.10766935348510742f, 1.0629868507385254f, 1.841042399406433f, -0.5686787366867065f, -1.44584059715271f, -0.2561878263950348f, 1.1728384494781494f, -0.428275465965271f, -0.2985764443874359f, 0.14890146255493164f, 1.661197304725647f, -2.622263193130493f, -0.43193310499191284f, -0.3783835768699646f, 0.23239530622959137f, -0.7321792840957642f, -0.40373992919921875f, 1.2062691450119019f, -0.6938396096229553f, -1.5164529085159302f, 2.0242106914520264f, -0.8075132966041565f, 0.09412498027086258f, 0.27298247814178467f, 0.5441054701805115f, -2.4758286476135254f, 0.21060240268707275f, 0.2536042630672455f, -0.7318422794342041f, 0.05695872753858566f, 0.15881912410259247f, 0.6510986685752869f, 0.020065616816282272f, 0.8478670120239258f, 2.0752015113830566f, -0.38090255856513977f, -0.9460003972053528f, -0.4590383470058441f, -0.6942368745803833f, 0.30009862780570984f, -1.389849305152893f, 1.2990328073501587f, -1.8040062189102173f, 1.837660551071167f, -0.7219073176383972f, -0.3056040108203888f, 0.9780712127685547f, 0.9052547812461853f, -0.901805579662323f, -0.2280527949333191f, -0.948022723197937f, -0.15365807712078094f, 1.032126545906067f, 0.17955327033996582f, -0.2721782922744751f, 0.15856090188026428f, 1.1139295101165771f, -0.5713488459587097f, -0.9870969653129578f, -1.6782877445220947f, 1.8284801244735718f, -0.4982566237449646f, 0.47082117199897766f, -0.591300904750824f, -0.41513174772262573f, 0.5321887731552124f, 1.317676305770874f, -0.7898051142692566f, -0.5088178515434265f, 0.23661386966705322f, 1.2172428369522095f, 0.2148703932762146f, 1.2481820583343506f, 2.4610743522644043f, 0.14153295755386353f, 1.837676763534546f, -0.2276618480682373f, 0.2129386067390442f, 0.8061029314994812f, 0.7021245956420898f, -0.6485925316810608f, -0.47295162081718445f, 2.2190706729888916f, -1.868390679359436f, -1.3732751607894897f, -1.013265609741211f, 1.365964651107788f, 1.1893959045410156f, -2.7800850868225098f, -1.0310310125350952f, -0.702293336391449f, 0.8099242448806763f, 0.8948581218719482f, -0.40090087056159973f, -0.08886078000068665f, -0.9508647918701172f, 0.28014296293258667f, 0.06467004120349884f, -2.057403087615967f, 1.1129963397979736f, -0.7580298185348511f, 0.0977560505270958f, 0.8891482949256897f, 0.3460270166397095f, -0.11394428461790085f, 1.4566701650619507f, -1.1309462785720825f, 1.2330803871154785f, 0.8789023160934448f, -2.017490863800049f, 0.856982946395874f, 0.04652906581759453f, -0.3498983681201935f, -0.8770002722740173f, -0.38342466950416565f, -0.05906866118311882f, 0.7605865001678467f, -0.9748004078865051f, 0.9013839960098267f, -0.43347686529159546f, -1.508413314819336f, 2.2633509635925293f, 0.88945072889328f, 0.7355136275291443f, -0.6224141120910645f, 0.2808963656425476f, 0.1529616415500641f, 0.09397149085998535f, 0.8745917677879333f, -1.226547122001648f, 1.3209213018417358f, 1.0341976881027222f, -0.49460795521736145f, -3.3865373134613037f, 2.0950191020965576f, 0.2728019058704376f, -0.030417079105973244f, 0.08612233400344849f, -1.7298319339752197f, 0.08109258860349655f, 1.2689827680587769f, -0.6729629635810852f, 0.8545035123825073f, -1.1793856620788574f, -0.9540548324584961f, -0.969673752784729f, 0.3141362965106964f, 1.2000207901000977f, -0.1831716001033783f, -0.18045181035995483f, -0.1034746766090393f, -0.10869846493005753f, 0.5782245993614197f, 1.7499538660049438f, 0.002034955658018589f, 0.36500951647758484f, -0.8946718573570251f, -0.219259113073349f, -0.47011202573776245f, -0.9500816464424133f, -0.6808398962020874f, -0.9808987379074097f, -0.7519583702087402f, -1.0324863195419312f, -1.8080289363861084f, 0.21794334053993225f, 0.6857624053955078f, 0.43733805418014526f, -1.9561761617660522f, 1.2915894985198975f, -1.6977379322052002f, -1.2025177478790283f, -1.646543025970459f, -1.3370387554168701f, 1.169712781906128f, -0.5579738020896912f, 0.38680657744407654f, -0.4910755455493927f, -0.32521501183509827f, 0.13093096017837524f, 0.624261200428009f, 0.6792762279510498f, 0.3959449231624603f, 0.66949063539505f, -0.19622361660003662f, -0.32004305720329285f, 0.20433473587036133f, -1.9621531963348389f, -0.5470468401908875f, 1.5721423625946045f, 1.0094727277755737f, 0.4029425382614136f, 1.6145533323287964f, 0.4475519359111786f, 0.5499113202095032f, 0.21999366581439972f, -0.019641423597931862f, 0.20199595391750336f, -0.305207222700119f, -0.11879883706569672f, 0.8017854690551758f, -0.3870505392551422f, 1.560797929763794f, 0.04964430257678032f, -0.33324524760246277f, 0.7588040828704834f, -1.5863133668899536f, -0.6730678081512451f, -1.2264569997787476f, 1.3401838541030884f, -1.162084698677063f, 1.4780147075653076f, 1.5603762865066528f, -0.3231433033943176f, 2.119018793106079f, 0.7877871990203857f, -1.9625552892684937f, -0.526032567024231f, 2.639968156814575f, 1.171855092048645f, 1.2722933292388916f, 0.4042589068412781f, -1.5217982530593872f, -0.5133728981018066f, -1.3806458711624146f, -0.5332193970680237f, 1.0821458101272583f, 0.6407410502433777f, -1.729612946510315f, -2.14782977104187f, 0.4973359704017639f, -1.268904685974121f, -0.24232769012451172f, -1.0745846033096313f, -0.6819858551025391f, 1.0070197582244873f, -0.8279359936714172f, 2.587069511413574f, -0.7386665940284729f, 0.3296529948711395f, 2.8114237785339355f, -0.879487931728363f, -0.33060166239738464f, 0.6457027792930603f, 0.17840361595153809f, -1.1289931535720825f, -0.3602311313152313f, -2.383305549621582f, 1.3506853580474854f, -0.4065081775188446f, -0.30150434374809265f, -1.5348396301269531f, 1.6021355390548706f, 0.6893559694290161f, 0.033526014536619186f, -0.2977294325828552f, -1.1700760126113892f, 1.5237687826156616f, 1.2984881401062012f, 0.6588833928108215f, -0.8222662210464478f, -0.14632517099380493f, 0.7788861393928528f, 0.6403340101242065f, 0.8075235486030579f, 0.5731524229049683f, 1.3903805017471313f, -0.23513318598270416f, -0.5386273264884949f, -0.8155969977378845f, -0.1509992778301239f, -0.5020706653594971f, 0.8036778569221497f, -1.1333380937576294f, -0.7435185313224792f, -0.7013434171676636f, -0.42178988456726074f, -1.5332636833190918f, -1.9342981576919556f, 0.3510490953922272f, -0.6404718160629272f, 0.019135747104883194f, -0.7149955034255981f, -0.7025302648544312f, 0.7700265645980835f, -0.7571792006492615f, -1.0642281770706177f, 1.9611531496047974f, -0.4390997588634491f, 0.8920031785964966f, 1.8720558881759644f, 0.24501417577266693f, 0.7017510533332825f, 0.9099079370498657f, -0.7323962450027466f, -0.6663155555725098f, -0.3277064561843872f, -1.439155101776123f, -1.2797472476959229f, -0.19622953236103058f, 0.19903564453125f, -0.48808231949806213f, -0.6568363904953003f, -0.4720822274684906f, 0.35971710085868835f, 0.5307507514953613f, 1.0079243183135986f, 1.6095112562179565f, 1.6016820669174194f, 0.020138248801231384f, 0.573236346244812f, 0.37203314900398254f, 0.22609540820121765f, -0.754463255405426f, -0.3379971385002136f, 0.41659975051879883f, 0.22890278697013855f, -0.5443984270095825f, 2.050001859664917f, 0.06459449976682663f, 1.448185920715332f, -0.20686893165111542f, 1.5211719274520874f, -1.2341058254241943f, -0.7871248126029968f, -0.7268313765525818f, -0.26047298312187195f, -0.1815493106842041f, 0.40858983993530273f, 0.21912771463394165f, -0.45708730816841125f, -0.651996910572052f, -0.32107922434806824f, 0.20424777269363403f, -0.8860291838645935f, 1.0496968030929565f, 0.5568186640739441f, 0.13751773536205292f, -0.15331187844276428f, 1.2169883251190186f, 1.7785981893539429f, -0.709164023399353f, -0.9840890765190125f, -0.17089858651161194f, 0.2695762813091278f, 0.3193606436252594f, -0.6905565857887268f, 0.3500501811504364f, -0.32124459743499756f, -0.25051894783973694f, 2.1708250045776367f, -1.229917049407959f, 0.3085583448410034f, -0.4612124264240265f, -1.0031081438064575f, -3.3694987297058105f, 0.7832650542259216f, -0.34354400634765625f, 0.34807446599006653f, -1.7184555530548096f, -0.8929248452186584f, -0.7456709742546082f, 0.1508672833442688f, -0.7863383889198303f, -0.350765198469162f, -0.04678618907928467f, -0.6908267736434937f, -0.38857290148735046f, 0.04591083899140358f, -0.3794901967048645f, 1.5770217180252075f, -0.042833227664232254f, 1.353956699371338f, -1.124684453010559f, -0.17863909900188446f, 0.20412735641002655f, 0.9273927807807922f, -0.32438376545906067f, 0.6851024031639099f, 1.0722298622131348f, -1.000272512435913f, -0.8011578917503357f, -1.122711181640625f, -0.7824243903160095f, -0.5122944712638855f, -0.8242338299751282f, -1.0234198570251465f, -1.3928337097167969f, -0.4097520112991333f, 0.8509801030158997f, 0.8881950378417969f, 0.5827952027320862f, 0.785405158996582f, 1.5136889219284058f, 1.251064419746399f, -1.9372137784957886f, -1.2051048278808594f, 1.1982215642929077f, -1.376950740814209f, 1.0670830011367798f, 0.507300615310669f, 0.4764442443847656f, 0.9100666046142578f, -1.501094937324524f, -0.11335238814353943f, -1.2001420259475708f, -0.7933986186981201f, -2.7309763431549072f, -0.014749551191926003f, -0.04001638665795326f, -0.11494863778352737f, 1.458335041999817f, 0.4836253225803375f, 1.7024855613708496f, -0.9683796763420105f, 1.086437702178955f, 1.8661012649536133f, 1.4962280988693237f, 1.022226333618164f, 1.5485637187957764f, -0.08855976164340973f, -0.88787841796875f, -0.6835475564002991f, -0.015076662413775921f, -0.030384592711925507f, -0.8306879997253418f, -1.7835168838500977f, -1.239241361618042f, 0.9374039173126221f, -0.12884187698364258f, 0.9388713836669922f, 0.2919924259185791f, 1.7478625774383545f, -0.8145643472671509f, 0.9247024655342102f, -1.5529402494430542f, 0.13943159580230713f, 1.0945836305618286f, 1.0349615812301636f, 0.03250877186655998f, 0.9967503547668457f, 0.6327362060546875f, 0.7831454873085022f, 0.5098334550857544f, 1.5633010864257812f, -0.46283742785453796f, 0.1497601568698883f, 0.6901775598526001f, -0.38219600915908813f, 0.4878943860530853f, 0.04518410563468933f, 1.9969353675842285f, -0.6955253481864929f, 0.5120656490325928f, -0.9795308709144592f, 2.3521711826324463f, 0.41813287138938904f, -0.1308819055557251f, -0.4394487142562866f, 1.6976182460784912f, 0.813541829586029f, -0.6087417602539062f, 0.9832689762115479f, 2.428079843521118f, 1.1440294981002808f, -0.101145900785923f, 0.09917446225881577f, 0.3814586102962494f, 1.6723263263702393f, -0.4240947961807251f, 1.8199645280838013f, -0.39397287368774414f, -0.4608776867389679f, 0.51866215467453f, 0.7784189581871033f, -1.5908970832824707f, 1.9877948760986328f, 0.02497357502579689f, -0.43002068996429443f, 1.0171878337860107f, -0.548503041267395f, 1.6083859205245972f, 0.5863268971443176f, 0.15758942067623138f, -0.9828112125396729f, 0.18478159606456757f, 1.4150960445404053f, 0.4230518639087677f, 0.5434484481811523f, -1.2116377353668213f, 0.1757367104291916f, 0.25499454140663147f, 0.8070029616355896f, -0.11587464064359665f, 0.45107510685920715f, -0.3600947856903076f, -1.0252388715744019f, 0.37429946660995483f, -0.3791492283344269f, 2.650177001953125f, -1.6031533479690552f, -0.47765374183654785f, -0.5845633149147034f, -0.21724168956279755f, -0.5731971859931946f, 0.37175968289375305f, -0.4610559642314911f, 2.174776315689087f, 0.021914366632699966f, 0.1974239945411682f, -0.7603867650032043f, -0.693874180316925f, -1.0808472633361816f, -0.06485684216022491f, -0.010432514362037182f, 1.2084461450576782f, -0.9010456800460815f, -1.3052830696105957f, -0.12543383240699768f, 1.2808754444122314f, 0.6577393412590027f, 0.46201997995376587f, -0.5072460770606995f, -0.9180205464363098f, 0.0884491354227066f, -1.0534428358078003f, 0.2707360088825226f, -0.25277644395828247f, 0.016658902168273926f, -0.2014150619506836f, 0.05029870569705963f, 1.4853851795196533f, 0.8810616731643677f, 0.9759232997894287f, 0.807918131351471f, -1.3484669923782349f, 1.1456364393234253f, 0.31596988439559937f, -0.4552096426486969f, -0.18135374784469604f, -0.3125359117984772f, 2.018014669418335f, -0.44383805990219116f, -0.7552081942558289f, -1.7850168943405151f, -0.07548937201499939f, -0.5207358598709106f, -1.048349142074585f, 0.8424208760261536f, -0.9976255297660828f, 0.7335740327835083f, 1.2276335954666138f, -0.7577790021896362f, 1.622873306274414f, 1.292340874671936f, -0.26265472173690796f, -0.8315040469169617f, 1.0420149564743042f, 1.154740810394287f, 0.31027355790138245f, -0.6142987608909607f, 1.667606234550476f, 0.014784318394958973f, -1.3251757621765137f, -0.8346104025840759f, -0.2093493789434433f, 0.2113834023475647f, -0.15933676064014435f, 1.438138484954834f, -0.8965448141098022f, 0.4655975103378296f, -0.7802034020423889f, -0.7512221336364746f, -0.20733071863651276f, 0.41877850890159607f, 1.7955039739608765f, -0.6994057297706604f, -0.4367755055427551f, 0.34429505467414856f, -0.5315167307853699f, -1.7878962755203247f, 0.5759621858596802f, -0.37107211351394653f, -0.02148612029850483f, -0.3246121108531952f, 0.7541995644569397f, -1.7302361726760864f, -0.6404775977134705f, -0.024382900446653366f, -0.7490391731262207f, -2.1543679237365723f, -0.15692853927612305f, 0.09731115400791168f, 0.2538682818412781f, 1.5049229860305786f, 1.313053011894226f, -0.7395356297492981f, -1.813647985458374f, -0.2881675958633423f, -0.4023030996322632f, 0.7287482619285583f, -0.49943476915359497f, 0.7028453350067139f, 0.34216001629829407f, -1.3413481712341309f, 0.41704463958740234f, -0.8712550401687622f, -0.16223306953907013f, -0.9481205940246582f, -0.6534963846206665f, -0.3585118353366852f, -0.817564845085144f, 0.3736099898815155f, 0.9122175574302673f, 0.6781302690505981f, 0.055852312594652176f, -1.3394618034362793f, -0.30016884207725525f, 0.15999306738376617f, -0.6769115328788757f, 0.7504494786262512f, 0.216787651181221f, 0.23502039909362793f, 1.8141969442367554f, -1.5329066514968872f, -0.3652367889881134f, -1.5126714706420898f, 0.7930607199668884f, -1.0631688833236694f, 0.9984860420227051f, 1.037604570388794f, -0.04009038954973221f, -1.5908479690551758f, 0.30711954832077026f, 1.1175891160964966f, -1.345697283744812f, 1.0572247505187988f, 1.3808448314666748f, -0.366502046585083f, -0.08894097805023193f, -0.662988007068634f, -0.291555255651474f, 0.10108345001935959f, 1.0123193264007568f, -0.815822958946228f, -0.18586060404777527f, -0.7641056776046753f, -0.3749435842037201f, -0.8628830313682556f, -0.8098255395889282f, -0.21431559324264526f, -1.1151552200317383f, 0.14144384860992432f, -0.17559614777565002f, -0.7265163660049438f, -0.5217191576957703f, 0.37568187713623047f, 0.5935606956481934f, 0.20674307644367218f, 1.4827877283096313f, 0.9587778449058533f, -0.3417870104312897f, -1.0058722496032715f, -1.0774322748184204f, -2.2288825511932373f, 0.8113047480583191f, 0.13623955845832825f, -0.30091696977615356f, -0.2654571533203125f, -1.8403681516647339f, 0.1881687343120575f, -1.4522165060043335f, 1.4667081832885742f, 0.9349684119224548f, 0.25753054022789f, 0.9158172607421875f, -0.5319824814796448f, 0.6224501132965088f, 0.12310407310724258f, -0.17781615257263184f, 2.9330546855926514f, 0.4627670347690582f, -0.6264359354972839f, -1.3771589994430542f, -1.6635388135910034f, -0.2668311595916748f, -1.2189315557479858f, 0.14258776605129242f, -0.31570670008659363f, 1.423683524131775f, 1.0125263929367065f, -0.7147458791732788f, 0.3446122407913208f, 0.8802022337913513f, -0.7164220213890076f, -1.3230516910552979f, 1.0481867790222168f, -1.7053356170654297f, 0.600459098815918f, 1.8295810222625732f, -0.340763121843338f, 0.34682390093803406f, -0.1821630895137787f, -0.12917080521583557f, 0.7775542140007019f, 0.3030397295951843f, -0.8394461274147034f, 0.4900873601436615f, 0.9002130031585693f, -1.4860044717788696f, 0.9374179840087891f, -0.5453174114227295f, 0.485347181558609f, 0.6994611024856567f, -0.28342336416244507f, -0.11592065542936325f, 0.22056539356708527f, -0.8155755400657654f, 1.1422392129898071f, 0.16099725663661957f, 1.1248847246170044f, 0.09503551572561264f, 0.6680463552474976f, 0.22494570910930634f, -0.8516902327537537f, 0.4524131119251251f, -0.7356429100036621f, 1.461809754371643f, 0.056686531752347946f, -0.6240271329879761f, -0.3087240159511566f, 0.18612052500247955f, 0.026730623096227646f, -0.8790956139564514f, 1.3807705640792847f, -0.6990379691123962f, -1.2263665199279785f, -1.9690755605697632f, 2.3512303829193115f, 0.35396817326545715f, -0.10606183111667633f, -2.408507823944092f, -1.2640177011489868f, -0.4252816140651703f, 0.24079595506191254f, -0.24080689251422882f, -0.6830735802650452f, 0.28334593772888184f, -0.7549530863761902f, 0.685922384262085f, -0.28879889845848083f, 0.16147172451019287f, -0.18717071413993835f, 0.7061312794685364f, 0.12637053430080414f, -0.2261367291212082f, -0.40762385725975037f, -0.6097135543823242f, -1.347667932510376f, 0.8310710787773132f, 0.23480524122714996f, 0.23941050469875336f, -0.15233780443668365f, 1.35030198097229f, 0.3163374066352844f, -0.686985969543457f, 0.31536293029785156f, -1.2173928022384644f, -0.23235148191452026f, -0.11819366365671158f, -0.7065277099609375f, 0.8570377826690674f, -0.6065717935562134f, -2.5028295516967773f, -0.5181497931480408f, -0.3955882489681244f, -0.007207863032817841f, -1.8654048442840576f, -1.2592604160308838f, 0.44495517015457153f, 1.5878770351409912f, -1.009257197380066f, 2.5517168045043945f, 0.27151748538017273f, -0.45277684926986694f, 0.7783017754554749f, 0.5624421238899231f, -1.7339200973510742f, 0.1151348203420639f, 0.5969645380973816f, 1.4009592533111572f, 1.26650869846344f, 0.5665708780288696f, 1.4182316064834595f, 0.8785027265548706f, 1.713185429573059f, 0.5062318444252014f, 0.9417456984519958f, 1.1770522594451904f, -0.8855880498886108f, -0.897435188293457f, -0.4500139355659485f, -1.2818336486816406f, -0.09055130183696747f, 1.568543553352356f, 0.6288840770721436f, -0.13983508944511414f, -0.8341673016548157f, -1.0642942190170288f, -0.4966285824775696f, -0.8965734243392944f, 0.2338687777519226f, 0.13322870433330536f, -1.087693452835083f, -0.6877674460411072f, -0.034093379974365234f, -1.2961610555648804f, 1.018326997756958f, 2.094313621520996f, 0.22312399744987488f, 0.04033441096544266f, -0.11042066663503647f, 1.209588646888733f, 0.8461918830871582f, 1.0998334884643555f, -2.8670525550842285f, 0.43270260095596313f, 2.3624584674835205f, -0.9460180401802063f, 0.40504685044288635f, 0.21921862661838531f, -1.154799461364746f, 1.519834041595459f, -1.8380980491638184f, 1.218536376953125f, -0.1873958259820938f, 1.0788236856460571f, 0.14058129489421844f, -0.9915911555290222f, -1.5487273931503296f, -1.3351367712020874f, -1.668448805809021f, -1.84798002243042f, 0.02599930204451084f, 0.7676464319229126f, -1.9874215126037598f, 0.9982309937477112f, 0.003224122803658247f, 0.07818755507469177f, -1.0705227851867676f, -1.0210168361663818f, 1.7321865558624268f, 0.5420076847076416f, 0.3053724765777588f, 1.1364459991455078f, 0.6844477653503418f, 0.2691081464290619f, 0.8210340738296509f, 0.27511492371559143f, -0.38933637738227844f, -0.20226091146469116f, -0.19409379363059998f, -0.07234596461057663f, -0.7745715975761414f, 0.2601335644721985f, 1.2001798152923584f, -1.078745722770691f, 1.2962092161178589f, -0.18015773594379425f, -1.9301296472549438f, 0.28462305665016174f, 0.7962566614151001f, -0.26019519567489624f, 0.30697566270828247f, -0.7991265654563904f, 0.6080788373947144f, -0.8028495907783508f, -0.46709370613098145f, 0.5347719192504883f, 1.8560413122177124f, -0.11687781661748886f, 0.6693155169487f, -0.06095214933156967f, -0.1428632140159607f, -1.3090907335281372f, 0.03258459270000458f, 1.4419121742248535f, -0.6330651044845581f, -0.03063541278243065f, 1.5843851566314697f, -0.467591792345047f, -0.03230065852403641f, 0.7200000286102295f, -0.988913357257843f, -1.0607333183288574f, 0.7062268257141113f, -0.2287754863500595f, -0.030369358137249947f, 0.7900131344795227f, -0.22298280894756317f, 1.508110761642456f, 0.599722146987915f, -0.33899757266044617f, 0.1548950970172882f, -0.2794382572174072f, 0.8708680272102356f, 0.7626234889030457f, 0.6961768865585327f, 0.44085901975631714f, -0.4885202944278717f, 0.7180350422859192f, 1.8486416339874268f, -2.431525230407715f, -0.4701055586338043f, 0.4380730092525482f, 0.7702670693397522f, -0.14850954711437225f, 0.24941392242908478f, 0.539443850517273f, 0.058982912451028824f, 0.28753769397735596f, -0.6253249645233154f, 1.5168472528457642f, -0.610085129737854f, -1.8375158309936523f, 1.8140860795974731f, 0.9514070153236389f, -1.2033350467681885f, -0.5142884254455566f, -0.8533352017402649f, -0.5986783504486084f, -0.2986593544483185f, -0.7190134525299072f, -1.4356969594955444f, 0.6340908408164978f, -0.5407924652099609f, 1.3264530897140503f, 0.29634353518486023f, 0.17156629264354706f, 0.8127211332321167f, -0.636906087398529f, -1.1077786684036255f, 1.5005162954330444f, 0.5292592644691467f, -1.5582870244979858f, -0.22035345435142517f, -0.8878109455108643f, -0.23165416717529297f, -0.48117324709892273f, -0.8784040212631226f, 0.039074741303920746f, 0.03500647097826004f, -1.3603416681289673f, -0.3714776337146759f, -0.040006835013628006f, 0.5846149921417236f, 0.5806360840797424f, 0.7450182437896729f, 1.4870532751083374f, -0.051454752683639526f, 0.2527291476726532f, 0.7950483560562134f, 1.2641831636428833f, -0.0403965562582016f, -0.19124414026737213f, -0.4378063976764679f, -0.009368259459733963f, 0.25679612159729004f, -0.003075298387557268f, -1.1646342277526855f, -0.46010419726371765f, 0.18153296411037445f, 0.18499712646007538f, -0.23162956535816193f, 2.05483078956604f, 0.4368106722831726f, 1.1093194484710693f, 1.684390902519226f, -0.11415659636259079f, 0.37000924348831177f, 0.2512214779853821f, 1.1637510061264038f, -0.876596987247467f, -0.6737048625946045f, -0.652714192867279f, -0.753913164138794f, -0.10021805018186569f, -2.3608896732330322f, 2.2464849948883057f, -1.1070212125778198f, 1.026766061782837f, 0.2865336239337921f, -0.10950178653001785f, 2.380542039871216f, -0.41917189955711365f, 0.6831162571907043f, 1.7866971492767334f, -0.022699790075421333f, 0.7491136193275452f, -1.1757766008377075f, 0.2672518789768219f, -0.22947388887405396f, 0.5409282445907593f, 0.6939815282821655f, 1.8563321828842163f}; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/kernel/matmul.c deleted file mode 100644 index a513aa38c..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-afterK/kernel/matmul.c +++ /dev/null @@ -1,613 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. - -#include "matmul.h" - -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) -{ - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); - uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); - uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; - uintptr_t c00 = (uintptr_t)(C); - uintptr_t c01 = c00 + operand_bytes; - uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); - uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; - uintptr_t loop_counter; - const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - const uintptr_t rewind_b0_bytes = operand_bytes * TE; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; - - asm volatile( - ".Lmatmul_block_%=:\n" - "mv %[tss], x0\n" - // K-Group 1 - "msetmtype %[mtype], %[vtype]\n" // set sew=16, set tm=TE, tk=2, twiden=1 - "li %[loop], 16\n" - "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE - // we dont need preload - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - "9:\n" - "mv %[loop], %[middle_k_groups]\n" - "beqz %[loop], 3f\n" - "1:\n" - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 1b\n" - "3:\n" - - // K-Group N-1 & K-Group N - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v9, v25\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" - "vle32.v v13, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v29, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v15, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v13, v29\n" - "vle32.v v31, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" - "vle32.v v16, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v17, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "vle32.v v18, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v19, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v20, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v21, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v22, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v23, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v0, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v1, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vle32.v v2, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v3, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v4, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v5, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v6, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v7, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v29, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vle32.v v31, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Complete the block before changing any A/B/C panel pointer. The - // former cross-block path deferred this mt8 store while beginning the - // next block, but its FP32 register preload was incomplete. - "lui %[tss], 0x40000\n" - "li %[loop], 16\n" - ".Lstore_mt8_%=:\n" - "vtse32 %[tss], (%[c10])\n" - "add %[c10], %[c10], %[c_stride]\n" - "addi %[tss], %[tss], 1\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], .Lstore_mt8_%=\n" - - // block transition - "vtzero mt0\n" - "addi %[blocks], %[blocks], -1\n" - "beqz %[blocks], 8f\n" - - // Advance horizontally until the N panel is exhausted. At the row - // boundary, advance A by two tiles and rewind B to its first pair. - "addi %[cols], %[cols], -1\n" - "beqz %[cols], 4f\n" - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "j 5f\n" - "4:\n" - "sub %[cols], %[b1p], %[b0p]\n" - "add %[a0p], %[a0p], %[cols]\n" - "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" - "addi %[loop], %[loop], -1\n" - "6:\n" - "sub %[b0p], %[b0p], %[cols]\n" - "sub %[b1p], %[b1p], %[cols]\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[loop], x0, %[loop]\n" - - "5:\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "j .Lmatmul_block_%=\n" - - // Legacy overlapped cross-block schedule retained below for assembly - // comparison, but no control-flow path enters it. - // vtfmm mt0 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v0, v24\n" - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v16, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" - "vle32.v v17, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vle32.v v18, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v19, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v8, v16\n" - "vle32.v v20, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" - "vle32.v v21, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v22, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" - "vle32.v v23, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "mv %[tss], x0\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "j 9b\n" - - // Final block: compact 16-row mt8 store loop. - "7:\n" - "li %[loop], 16\n" - "2:\n" - "vtse32 %[tss], (%[c10])\n" - "add %[c10], %[c10], %[c_stride]\n" - "addi %[tss], %[tss], 1\n" - "addi %[loop], %[loop], -1\n" - "bnez %[loop], 2b\n" - "8:\n" - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - [blocks] "+r"(block_counter), - [cols] "+&r"(col_counter), - [loop] "=&r"(loop_counter) - : - [c_stride] "r"(c_stride), - [operand_bytes] "r"(operand_bytes), - [mtype] "r"(matrix_mtype), - [vtype] "r"(matrix_vtype), - [rewind_b0_bytes] "r"(rewind_b0_bytes), - [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/CMakeLists.txt deleted file mode 100644 index 8188904a0..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(rolled-store-interleave "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/data/data_64_64_64.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/data/data_64_64_64.h deleted file mode 100644 index 49d1b3900..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/data/data_64_64_64.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Auto-generated by script/gen_data.py — do not edit by hand. - -#include "layer.h" - -const vme_gemm_layer gemm_l = { - .M = 64, - .N = 64, - .K = 64, - .TM = 8, - .TN = 8, -}; - -// C[M*N] row-major fp32 golden reference, C = A*B -static float gemm_C_dram[64*64] __attribute__((section(".data"))) = -{11.479634284973145f, -6.046323299407959f, -4.815614700317383f, -2.040816068649292f, -8.940674781799316f, 1.9331291913986206f, 4.764830112457275f, 14.248414993286133f, -3.9397127628326416f, -2.1485483646392822f, -0.8390139937400818f, -1.3708610534667969f, 13.092900276184082f, 3.2064008712768555f, 5.635993480682373f, 0.7455204725265503f, 7.919027805328369f, -11.349638938903809f, -4.450965404510498f, 15.0212984085083f, 0.6696339845657349f, -0.1861145943403244f, 4.869348049163818f, 3.4079155921936035f, -3.9942312240600586f, 8.851547241210938f, 8.788155555725098f, 6.571914127562195e-05f, 9.69917106628418f, -8.461928367614746f, -8.80200481414795f, 4.7893218994140625f, -0.18487417697906494f, -11.189379692077637f, -12.456897735595703f, -3.528578758239746f, 6.636075019836426f, 4.215647220611572f, -2.5566506385803223f, -2.2709872722625732f, 24.085134506225586f, -3.519670009613037f, -1.5794446468353271f, 13.206070899963379f, -9.0339937210083f, -0.49932846426963806f, 8.33430290222168f, -3.4920732975006104f, 24.13520622253418f, -2.866286516189575f, -5.177119255065918f, 8.296358108520508f, -2.486847162246704f, 7.566592693328857f, -18.43720817565918f, 17.894702911376953f, -17.132667541503906f, 4.636127471923828f, -15.027854919433594f, 14.163813591003418f, -16.567644119262695f, 4.0814080238342285f, -19.760257720947266f, 0.5436081290245056f, -1.0178847312927246f, -0.959050178527832f, -7.164222717285156f, 0.9985886216163635f, -8.26685619354248f, -2.2400271892547607f, 0.6488896608352661f, -3.7606098651885986f, 6.696377277374268f, 1.2368918657302856f, 0.3883592486381531f, -2.765044689178467f, 1.5367969274520874f, 5.624364852905273f, -1.1590087413787842f, 5.226413249969482f, 8.801167488098145f, 0.28941860795021057f, 7.781001091003418f, 1.1585612297058105f, -15.459006309509277f, -3.4766643047332764f, -8.453717231750488f, 10.173603057861328f, 3.6514530181884766f, 1.6193315982818604f, 5.127340793609619f, -0.9438349008560181f, -3.7507288455963135f, 13.667619705200195f, -10.900826454162598f, 11.15420913696289f, -2.949625253677368f, -2.746349811553955f, -16.438350677490234f, -17.064266204833984f, -4.720984935760498f, 4.893701553344727f, -2.473956823348999f, 4.502293586730957f, -5.264101982116699f, -8.454588890075684f, -5.186229705810547f, 5.584354877471924f, -2.95721435546875f, -3.410273790359497f, -6.588162899017334f, -0.8277148604393005f, -2.7242908477783203f, 4.025758266448975f, -1.4152922630310059f, 2.386408805847168f, 7.04514741897583f, -0.22987337410449982f, -0.45179229974746704f, 12.081316947937012f, -5.206918716430664f, 5.094143390655518f, -5.665627956390381f, 10.579627990722656f, -6.4196953773498535f, -8.07258415222168f, -19.385873794555664f, -8.815452575683594f, 1.0405519008636475f, -15.006485939025879f, -7.755399227142334f, 10.724888801574707f, 6.764276504516602f, 8.178961753845215f, 7.572741985321045f, 18.397531509399414f, -1.7755249738693237f, -1.9764132499694824f, 2.622986078262329f, -3.577617645263672f, -0.9461591839790344f, 14.711101531982422f, 8.183825492858887f, -7.469543933868408f, 14.31879711151123f, -6.517776012420654f, 2.1930079460144043f, 6.8929572105407715f, -10.022858619689941f, -1.7278424501419067f, 12.308111190795898f, 13.900723457336426f, 7.140420913696289f, 12.356964111328125f, 4.42258358001709f, -0.5004820227622986f, -1.5587430000305176f, 15.559115409851074f, 0.4262810945510864f, -2.8206470012664795f, 15.083824157714844f, -4.9072794914245605f, 2.5857160091400146f, 4.584408283233643f, -17.988496780395508f, -11.013274192810059f, 2.8712387084960938f, 4.990177631378174f, 0.6336867213249207f, 8.750150680541992f, 18.924428939819336f, 3.4662630558013916f, -14.959811210632324f, 3.8478240966796875f, 17.260263442993164f, -13.368204116821289f, 2.653244972229004f, -4.103200435638428f, -12.37991714477539f, -0.2966856062412262f, 5.892507076263428f, 6.881699562072754f, -3.230393648147583f, -10.482145309448242f, -13.298931121826172f, -6.5993547439575195f, 0.7129412293434143f, -2.0269880294799805f, 4.382270336151123f, -6.041990756988525f, -17.212663650512695f, -6.112694263458252f, 13.97390365600586f, -4.114153861999512f, -0.9799354672431946f, -4.399726390838623f, -5.032261848449707f, 0.5352955460548401f, -1.2309777736663818f, 10.451981544494629f, -5.048009872436523f, 3.677152633666992f, 11.969364166259766f, -0.417408287525177f, -3.176297426223755f, 5.826728343963623f, -5.306847095489502f, 10.880280494689941f, 12.627052307128906f, -5.021015644073486f, 0.382880836725235f, -12.032983779907227f, -2.4724607467651367f, -2.954784393310547f, 4.853247165679932f, -2.217081308364868f, -7.221776485443115f, 5.463408470153809f, -0.44735994935035706f, -10.677308082580566f, 1.9355524778366089f, -2.376770496368408f, 1.6620814800262451f, -12.09306526184082f, 9.559584617614746f, -1.3697234392166138f, -8.889398574829102f, 13.613365173339844f, -6.308327674865723f, -2.4649970531463623f, 2.388371467590332f, 4.159750938415527f, 0.5139855742454529f, 4.015327453613281f, -3.1259841918945312f, -1.0106412172317505f, 3.4432032108306885f, 4.1760029792785645f, -6.856860637664795f, -2.4840879440307617f, 0.686843752861023f, -1.0297985076904297f, 2.4013655185699463f, -3.5610408782958984f, 0.8839467167854309f, -2.944474458694458f, 12.640097618103027f, -3.0358707904815674f, -8.915057182312012f, 20.95890998840332f, 11.799171447753906f, 0.5034058690071106f, -1.9294973611831665f, -1.5430312156677246f, -1.1323392391204834f, 10.141955375671387f, -0.7313078045845032f, 3.43355655670166f, 12.49581241607666f, -0.004268331453204155f, 10.47712230682373f, 19.086078643798828f, -5.440404891967773f, -6.789121150970459f, 0.7849673628807068f, -0.7528467178344727f, -3.794801712036133f, -7.670317649841309f, 14.326471328735352f, -9.476102828979492f, 6.799417972564697f, -10.577319145202637f, -12.32802963256836f, 3.4096364974975586f, -24.734689712524414f, 3.53064227104187f, -1.0564911365509033f, 1.1934951543807983f, 2.728233814239502f, -2.273710012435913f, 1.6628460884094238f, 2.3249359130859375f, 3.8170626163482666f, 4.500376224517822f, -10.710893630981445f, 11.639716148376465f, -4.631977558135986f, 14.680305480957031f, 4.689661979675293f, -21.389917373657227f, 5.577933311462402f, 10.157344818115234f, 1.3816673755645752f, -9.77694034576416f, -1.59405517578125f, -5.5923357009887695f, 4.891682147979736f, -7.728332996368408f, -5.265920639038086f, 8.699220657348633f, -16.415210723876953f, 15.636664390563965f, -9.588492393493652f, -0.5692959427833557f, 2.7607104778289795f, 1.7100213766098022f, 5.899023532867432f, 6.637477874755859f, 1.3234360218048096f, 11.593015670776367f, 3.8447062969207764f, -12.39452075958252f, 17.31923484802246f, -4.310085296630859f, 1.3943078517913818f, 3.690248966217041f, 1.5341496467590332f, -0.16133688390254974f, -5.661871910095215f, -4.113314628601074f, 6.673490524291992f, -28.29741668701172f, -3.9005439281463623f, 5.377161979675293f, 2.418168067932129f, 14.741388320922852f, 12.219460487365723f, 2.1486709117889404f, -0.9644702672958374f, 2.6650185585021973f, -5.7412190437316895f, -11.613107681274414f, -4.728487014770508f, 4.203025817871094f, 4.285702705383301f, -6.733584880828857f, -1.751605749130249f, 1.8439090251922607f, 3.607456684112549f, -7.146326541900635f, 10.433805465698242f, 2.1755292415618896f, 7.192582607269287f, 2.093909502029419f, -12.907740592956543f, 10.749999046325684f, -0.37183743715286255f, -13.879500389099121f, -2.7690041065216064f, -0.5802304744720459f, -2.454869508743286f, 6.290034770965576f, -1.3150807619094849f, 1.3150523900985718f, -7.275416851043701f, -4.452425479888916f, 13.685830116271973f, -10.66769790649414f, 1.0413782596588135f, -11.05685043334961f, 18.703815460205078f, -1.3163670301437378f, 4.386673450469971f, 6.897695541381836f, -10.802721977233887f, -1.752117395401001f, 5.392110824584961f, -6.665993690490723f, 6.647964000701904f, 6.248779296875f, -8.761367797851562f, 3.014192581176758f, -12.200126647949219f, -4.44820499420166f, -6.547129154205322f, 5.838053226470947f, -3.972179889678955f, 4.548210620880127f, -0.38630470633506775f, 0.557646632194519f, -7.725645065307617f, -3.0316765308380127f, -10.36661148071289f, 9.558512687683105f, 1.3280283212661743f, -0.6749752759933472f, 2.5804808139801025f, 14.323273658752441f, 3.224461317062378f, -11.820540428161621f, -3.7925498485565186f, -8.334758758544922f, 0.4022352397441864f, 7.823636531829834f, -3.6518750190734863f, 2.0640387535095215f, -2.0144593715667725f, 1.261433482170105f, -12.39039421081543f, 1.648199200630188f, -8.816394805908203f, -6.666141986846924f, 4.299948692321777f, 11.934805870056152f, -9.07494831085205f, 8.816624641418457f, -7.657290935516357f, -1.1243833303451538f, 3.259331464767456f, 2.445486545562744f, 0.27485233545303345f, 9.906669616699219f, 10.861600875854492f, 1.530747413635254f, 3.6094765663146973f, 6.081133842468262f, -6.803634166717529f, 9.30950927734375f, -7.500870704650879f, -0.20945091545581818f, -4.56763219833374f, 9.062397956848145f, -3.900595188140869f, 0.46193817257881165f, -4.673060417175293f, -0.03734937310218811f, -7.745742321014404f, -1.5532246828079224f, -0.17770543694496155f, 7.410264492034912f, -2.503795862197876f, -4.5144267082214355f, 1.2131376266479492f, 1.3187705278396606f, -1.5181187391281128f, 1.0714176893234253f, 8.04094409942627f, 0.9295154213905334f, 4.887729167938232f, 4.386360168457031f, 2.0568528175354004f, -11.794127464294434f, -8.06806755065918f, 1.753397822380066f, -17.30348777770996f, -9.977322578430176f, 1.4055453538894653f, -3.8515894412994385f, -5.236893653869629f, 14.334654808044434f, -2.8135969638824463f, 12.197455406188965f, 5.414585113525391f, -3.6971359252929688f, 3.15084171295166f, -5.809675216674805f, 1.7624473571777344f, -1.7521796226501465f, -4.359520435333252f, -6.7524309158325195f, -6.430643558502197f, 4.465829372406006f, 5.363595008850098f, -0.05738918483257294f, -1.2110704183578491f, -12.731424331665039f, 7.597854137420654f, 5.500415325164795f, -6.625348091125488f, -4.435070991516113f, 16.017005920410156f, 6.0957231521606445f, 3.6745896339416504f, 18.56254005432129f, -6.415927886962891f, 7.9389967918396f, 2.5001425743103027f, 13.855697631835938f, 3.3746798038482666f, -13.604127883911133f, 0.829696774482727f, -0.6029493808746338f, 5.70452880859375f, 2.657468795776367f, -2.0609846115112305f, -5.867642402648926f, 9.778928756713867f, 4.533025741577148f, 5.282003402709961f, -1.0826342105865479f, -6.271111965179443f, 0.7187356352806091f, 8.167415618896484f, -17.741748809814453f, 5.170361518859863f, 1.9452539682388306f, 0.710178554058075f, 15.03416633605957f, 5.8055901527404785f, 9.560342788696289f, -7.244373798370361f, -7.103370189666748f, 6.701330184936523f, -13.6785306930542f, -7.435286998748779f, -1.4204037189483643f, 3.6168243885040283f, -1.2848598957061768f, 10.019153594970703f, -5.542983531951904f, 7.791118621826172f, 10.698689460754395f, 0.5794566869735718f, -5.135282039642334f, -0.725027859210968f, 6.094788551330566f, 3.9960646629333496f, 9.213926315307617f, -1.1690841913223267f, -11.310675621032715f, 4.797519207000732f, 2.9180209636688232f, 17.760066986083984f, -3.3779516220092773f, -5.8081278800964355f, 11.280855178833008f, -4.242335319519043f, -9.20323657989502f, 5.160378456115723f, 3.6303231716156006f, 8.061834335327148f, 0.7347631454467773f, 10.492819786071777f, -0.8277621865272522f, 15.720081329345703f, 6.086870193481445f, 2.3116986751556396f, -8.257061958312988f, -11.12254524230957f, -4.629530906677246f, -1.0439260005950928f, 2.503861665725708f, 4.953763484954834f, -7.421053886413574f, -4.31266975402832f, -0.7821304202079773f, -9.519991874694824f, -5.7066450119018555f, 2.1456873416900635f, 0.6137762665748596f, 9.2358980178833f, -4.393798351287842f, 0.7774427533149719f, -6.750569820404053f, 6.697755813598633f, 1.7473117113113403f, -5.084836959838867f, -0.3822762370109558f, 3.1108381748199463f, 1.4163764715194702f, -2.868486166000366f, -2.0308327674865723f, 12.085240364074707f, 0.32344046235084534f, -8.123509407043457f, -8.63557243347168f, 0.37889865040779114f, 5.865474700927734f, -2.006664752960205f, -4.2740278244018555f, -3.2797470092773438f, 0.7500452995300293f, 6.715641021728516f, 1.116511583328247f, -5.29896879196167f, 15.738234519958496f, -1.072593331336975f, 7.1444854736328125f, -0.17794132232666016f, 0.2486240416765213f, 10.500697135925293f, 5.418370246887207f, -2.850708484649658f, 5.028833866119385f, -1.3049635887145996f, -3.1064016819000244f, 7.260385036468506f, -0.2315407395362854f, 13.63786506652832f, -3.968128204345703f, 5.425351619720459f, 5.989855766296387f, 0.4256654977798462f, -11.24907112121582f, -13.982461929321289f, 3.0191447734832764f, -2.6573190689086914f, 0.18543148040771484f, 2.4465410709381104f, 7.224967002868652f, 9.639179229736328f, 11.72110366821289f, -14.941984176635742f, 2.158461809158325f, -9.967697143554688f, 7.731217384338379f, 0.2547948360443115f, 8.561541557312012f, -1.320813775062561f, -6.716103553771973f, -13.876091003417969f, -10.98348331451416f, 4.823574066162109f, 5.995814323425293f, 13.67184829711914f, -3.9956705570220947f, 2.217562675476074f, 8.181840896606445f, 11.416173934936523f, -7.419627666473389f, -2.8480899333953857f, -3.8081743717193604f, -9.326810836791992f, 2.7183923721313477f, 3.146589994430542f, -3.0829880237579346f, -5.814912796020508f, 8.27266788482666f, -2.7244033813476562f, 1.7461737394332886f, 12.851394653320312f, 6.015487194061279f, -11.382539749145508f, -10.727778434753418f, 1.8263652324676514f, 4.965327262878418f, -4.425846576690674f, 1.1692211627960205f, 0.7969360947608948f, 0.08354203402996063f, 6.083127975463867f, 5.9863152503967285f, -6.197132587432861f, -0.453395277261734f, -1.288467288017273f, -6.85847806930542f, 6.487435340881348f, -5.534632682800293f, 13.624343872070312f, 1.5271718502044678f, -1.4509201049804688f, -12.175165176391602f, -9.207465171813965f, 9.6865873336792f, -4.8576860427856445f, 5.351866245269775f, -8.007092475891113f, -1.1575658321380615f, 13.337957382202148f, 11.559327125549316f, -15.20022201538086f, 0.6360299587249756f, -16.20244789123535f, 7.8914031982421875f, 7.922462463378906f, 0.49626535177230835f, 0.6453051567077637f, 17.201900482177734f, 4.8184661865234375f, -9.721735954284668f, -9.466944694519043f, -1.581121563911438f, -3.897400379180908f, 13.486580848693848f, -2.706510305404663f, 2.2266478538513184f, -10.676257133483887f, -8.26251220703125f, -3.0972604751586914f, 6.262456893920898f, -7.870155334472656f, -13.106574058532715f, 1.9502202272415161f, 7.944813251495361f, 12.406501770019531f, -14.935750007629395f, 14.834214210510254f, -6.443024635314941f, -7.034519195556641f, 19.392358779907227f, -6.4235076904296875f, 5.952810287475586f, 4.202585697174072f, -6.166481018066406f, 6.54275369644165f, 4.269073009490967f, -4.20452356338501f, -11.693081855773926f, 0.5500400066375732f, -6.326947212219238f, -8.8986234664917f, -15.315683364868164f, 11.211211204528809f, -3.6820297241210938f, -4.241352081298828f, -0.7907126545906067f, 9.467594146728516f, 3.63688588142395f, -3.874994993209839f, -9.040021896362305f, 0.5441036224365234f, 14.560470581054688f, -4.352148532867432f, 2.122553586959839f, 7.5770955085754395f, 10.166125297546387f, -2.813676118850708f, -5.165591716766357f, -10.61868953704834f, 2.6967086791992188f, 1.273497462272644f, 16.388473510742188f, 12.926139831542969f, -18.8586483001709f, -2.5625576972961426f, -1.4167602062225342f, 3.1684563159942627f, -0.9735271334648132f, -12.724577903747559f, -9.471477508544922f, 1.9890105724334717f, -5.327760696411133f, 4.15165901184082f, -0.2690247595310211f, -5.091151237487793f, -8.251236915588379f, -13.286526679992676f, 11.11539363861084f, -2.1142094135284424f, -8.37338924407959f, -4.229764461517334f, 2.3841516971588135f, 9.676994323730469f, -16.521343231201172f, 21.009220123291016f, -14.100972175598145f, -18.230031967163086f, -2.3318724632263184f, 6.4106621742248535f, 5.1233930587768555f, 3.14448618888855f, -2.510714292526245f, 4.122291088104248f, -16.625221252441406f, -0.8044806122779846f, -7.060884952545166f, 15.047514915466309f, 1.5442057847976685f, -1.132941484451294f, 1.3845758438110352f, 5.088216781616211f, 5.57691764831543f, -5.411052703857422f, 15.526586532592773f, -8.33289623260498f, 2.9822065830230713f, -1.6032358407974243f, -16.77972984313965f, -8.319680213928223f, 1.4303581714630127f, 16.107824325561523f, -0.7335758805274963f, -10.483888626098633f, 6.441460132598877f, 4.506333827972412f, 2.4685215950012207f, 4.076876640319824f, -11.192659378051758f, -4.523487091064453f, -7.707409858703613f, 11.638382911682129f, -8.234586715698242f, 3.4906229972839355f, -9.156494140625f, -2.1987483501434326f, -14.089797019958496f, -0.26744771003723145f, 9.441929817199707f, 9.555951118469238f, 3.5714211463928223f, -13.786009788513184f, 2.148815870285034f, -6.62591028213501f, 7.7097487449646f, -1.0936174392700195f, 6.26248836517334f, 4.983580589294434f, -14.675965309143066f, -6.113205909729004f, 6.947558879852295f, -8.564501762390137f, 12.769579887390137f, -2.9595448970794678f, -2.199409008026123f, 4.292818069458008f, 0.014189623296260834f, -13.7341890335083f, 9.055596351623535f, 9.658831596374512f, 11.737518310546875f, -1.8660013675689697f, -7.643200397491455f, -7.508113384246826f, 8.38414192199707f, 5.401965618133545f, 13.07395076751709f, 3.3191146850585938f, 4.307673931121826f, 2.2782421112060547f, 1.9879189729690552f, -8.233000755310059f, -9.047646522521973f, 7.657324314117432f, 4.247537612915039f, 11.054131507873535f, -4.79965353012085f, -0.3028814196586609f, 3.658170461654663f, 6.45146369934082f, -2.8840818405151367f, 2.958085536956787f, 0.6611911654472351f, -15.455345153808594f, 14.44206428527832f, -1.5460273027420044f, 7.024033546447754f, -4.981692790985107f, -1.476488709449768f, -11.16106128692627f, 5.7634100914001465f, -4.065712928771973f, -7.169682502746582f, 6.687571048736572f, 9.344539642333984f, 0.3659030795097351f, 9.137345314025879f, -2.647665023803711f, -1.841318964958191f, 5.127373695373535f, -10.907052993774414f, 12.13635540008545f, 7.434513568878174f, -4.468003273010254f, 3.7723612785339355f, 6.410868167877197f, 4.488480091094971f, -4.279177665710449f, 7.486018180847168f, 2.041238784790039f, 0.8295110464096069f, 6.553818702697754f, 0.889700710773468f, 4.766284465789795f, -10.556770324707031f, -4.710302352905273f, -4.570557594299316f, 2.7720491886138916f, -0.9912987351417542f, -14.845259666442871f, 9.316060066223145f, 1.0089187622070312f, 8.025344848632812f, -1.9029171466827393f, 7.934247016906738f, -9.479381561279297f, 2.46075439453125f, 4.621790409088135f, -9.061838150024414f, 5.576177597045898f, 1.5385044813156128f, -0.02164754830300808f, 3.5387630462646484f, 8.792508125305176f, 11.135085105895996f, 0.09514058381319046f, 2.4890284538269043f, 6.469166278839111f, -12.828638076782227f, -11.913107872009277f, -12.835579872131348f, -0.7011198997497559f, 7.332045555114746f, 8.766676902770996f, -5.067351341247559f, -0.9870837926864624f, -12.046849250793457f, -8.263047218322754f, 7.80837869644165f, -2.4639174938201904f, 4.959366798400879f, 1.3135488033294678f, 0.6050440073013306f, 4.289178848266602f, -1.3765535354614258f, -9.210429191589355f, -2.441307544708252f, -13.704766273498535f, 1.7074304819107056f, 8.250596046447754f, -9.176819801330566f, -2.5442323684692383f, -10.301264762878418f, -9.052530288696289f, -10.033407211303711f, -6.968941688537598f, -2.9384915828704834f, 11.104891777038574f, 12.18455696105957f, -1.7053841352462769f, -10.236660957336426f, 13.148673057556152f, 5.770162105560303f, -14.608895301818848f, -3.367353677749634f, -1.997019648551941f, 0.6409828662872314f, 10.021282196044922f, 5.919330596923828f, -1.6916216611862183f, 8.11469841003418f, 15.174899101257324f, 1.4809973239898682f, 7.514713764190674f, -5.768565654754639f, 2.8586220741271973f, 0.785350501537323f, 6.117953300476074f, 12.343249320983887f, 11.014166831970215f, -3.0343480110168457f, -3.784940004348755f, -0.5094264149665833f, -3.813234567642212f, -1.6921602487564087f, 9.909290313720703f, 4.26685905456543f, -0.35572367906570435f, 13.063310623168945f, 0.9254683256149292f, -0.6383225321769714f, -4.169843673706055f, -0.2921450138092041f, -9.779097557067871f, 0.32207998633384705f, 0.46088171005249023f, -1.499398112297058f, -11.92739486694336f, 6.62190055847168f, 16.76767349243164f, 7.850483417510986f, -6.002956867218018f, -6.0743865966796875f, 8.590424537658691f, -8.350720405578613f, -7.048295497894287f, 8.3378267288208f, 10.439396858215332f, 4.051339149475098f, 7.447882652282715f, 2.115618944168091f, -10.832535743713379f, 3.544856071472168f, -10.704363822937012f, -1.747441053390503f, 3.1754229068756104f, -4.96966028213501f, -10.882390975952148f, 2.8613874912261963f, -14.762873649597168f, -3.866286516189575f, -9.634818077087402f, -4.488661289215088f, -16.146400451660156f, -8.561323165893555f, 10.843777656555176f, 7.5432963371276855f, 8.12264347076416f, -7.4483113288879395f, -10.192276000976562f, -0.18046212196350098f, 11.086979866027832f, 28.458358764648438f, -5.693641662597656f, 6.983304500579834f, 4.484742164611816f, -7.9314494132995605f, 5.449891567230225f, 3.5121002197265625f, -10.800859451293945f, 2.2634434700012207f, -10.603031158447266f, 6.369093894958496f, 21.34639549255371f, 5.682496070861816f, -6.500676155090332f, -3.933079719543457f, 9.044882774353027f, -1.2291513681411743f, 2.2405006885528564f, 7.146441459655762f, -5.252052307128906f, 13.324518203735352f, -2.54026198387146f, -4.696802139282227f, -1.9487093687057495f, 1.8994616270065308f, -12.246219635009766f, -6.6798505783081055f, -7.5826520919799805f, -4.574042320251465f, -1.7202016115188599f, 2.292769193649292f, 10.378962516784668f, 6.300224781036377f, 6.089559555053711f, 2.338552713394165f, 10.752300262451172f, -9.21757984161377f, 3.5770480632781982f, 1.6553243398666382f, -5.025013446807861f, -1.0492916107177734f, 2.3430588245391846f, -2.3533082008361816f, 13.465797424316406f, 8.130950927734375f, -17.104875564575195f, 3.76094913482666f, 3.0935189723968506f, 1.6557022333145142f, -1.1840074062347412f, 13.991741180419922f, 18.070926666259766f, -1.3191888332366943f, -5.403342247009277f, 3.2503833770751953f, 11.285150527954102f, 9.816794395446777f, -5.216005802154541f, -1.5604654550552368f, 6.930087089538574f, 2.469757556915283f, 0.5377483367919922f, -6.6069655418396f, -2.20501971244812f, -7.6572394371032715f, -12.394196510314941f, 4.9394073486328125f, -18.88083267211914f, 16.13848114013672f, 2.5295422077178955f, -6.287371635437012f, 3.2773492336273193f, 3.9482178688049316f, 1.5670056343078613f, -0.5477782487869263f, -9.703927993774414f, 7.892303466796875f, 16.467126846313477f, 1.2177623510360718f, -3.8086254596710205f, -0.10376394540071487f, 2.0969502925872803f, -13.377406120300293f, 11.43333625793457f, 0.33095118403434753f, -4.543916702270508f, -8.336254119873047f, 10.293583869934082f, -5.526627063751221f, 7.208284854888916f, 1.3287492990493774f, -0.00872433464974165f, -4.405787467956543f, 24.910831451416016f, 5.813371658325195f, -2.3747384548187256f, -1.58162522315979f, 7.138689041137695f, -0.8299044370651245f, 1.7968316078186035f, -0.16036289930343628f, 7.750216007232666f, -7.1681060791015625f, -1.6841415166854858f, 8.755945205688477f, -6.4495062828063965f, -3.929804801940918f, -11.690154075622559f, 5.407276630401611f, -19.410442352294922f, -7.46274471282959f, -6.528491497039795f, 11.476672172546387f, -10.896162986755371f, 9.078445434570312f, 4.966217994689941f, 2.7781200408935547f, 8.466453552246094f, 6.95072078704834f, -14.296183586120605f, 5.738059997558594f, -0.7061849236488342f, -3.4425599575042725f, -0.2813775837421417f, 4.374084949493408f, -2.8933258056640625f, -10.747404098510742f, -2.676332473754883f, 3.489074230194092f, 9.93961238861084f, 5.222956657409668f, -17.090818405151367f, 1.7347217798233032f, 13.138232231140137f, 8.740431785583496f, 6.423698425292969f, 1.796844244003296f, 2.321765661239624f, 6.327892303466797f, 0.3434549570083618f, 4.261251926422119f, 8.633129119873047f, -9.238373756408691f, 2.2492949962615967f, 2.5688412189483643f, -10.994841575622559f, -2.4493515491485596f, -3.7998733520507812f, -7.384821891784668f, -3.2261850833892822f, 17.528900146484375f, 3.3615753650665283f, 9.745475769042969f, -8.878006935119629f, 5.870270252227783f, -3.883683919906616f, -0.0326266773045063f, 7.409327030181885f, 10.064127922058105f, 17.231494903564453f, -4.299467086791992f, 0.6162081956863403f, -3.9329636096954346f, 3.457488775253296f, 1.6008586883544922f, 14.369483947753906f, -7.038842678070068f, 2.9360132217407227f, -11.51530933380127f, -6.849390029907227f, 13.802511215209961f, -3.17645263671875f, 8.121315002441406f, -4.379727840423584f, -0.7692107558250427f, -7.983859539031982f, -2.9673516750335693f, -16.916322708129883f, -16.98297882080078f, -10.993241310119629f, 6.7799153327941895f, -8.828645706176758f, -8.281076431274414f, 0.38319116830825806f, -6.442864418029785f, 5.842170715332031f, 2.7757022380828857f, -0.3426856994628906f, 11.318920135498047f, 9.262431144714355f, -17.823097229003906f, -11.620147705078125f, 0.7059615254402161f, 4.266175746917725f, 13.950201988220215f, -9.230637550354004f, -0.817978024482727f, 7.5885210037231445f, 15.164800643920898f, 7.9170756340026855f, -8.626565933227539f, -11.744421005249023f, -8.501404762268066f, -5.230059623718262f, -4.577203273773193f, -2.578476667404175f, 4.45864725112915f, 11.191590309143066f, 0.7849674224853516f, 2.8678131103515625f, 19.044151306152344f, -5.121172904968262f, 1.8655592203140259f, 5.731597900390625f, -4.264453411102295f, 0.31350159645080566f, 16.386018753051758f, -9.369375228881836f, -7.727303981781006f, -5.3810529708862305f, 4.404619216918945f, -5.699888229370117f, -5.798112869262695f, 0.13061971962451935f, 2.239311456680298f, 16.179100036621094f, -7.417810440063477f, 2.5456221103668213f, -3.236337900161743f, 1.1458196640014648f, -0.1307709664106369f, -0.8587997555732727f, -4.094841480255127f, -7.875479698181152f, -2.913313865661621f, 3.096707344055176f, 1.1844000816345215f, 11.496950149536133f, 4.843856334686279f, -4.238936424255371f, -1.1538751125335693f, -3.270214080810547f, 11.011395454406738f, 6.255121231079102f, 2.79670786857605f, -2.360741376876831f, 6.709340572357178f, -0.9215971827507019f, 4.753165245056152f, 11.404985427856445f, -11.04754638671875f, 5.086482048034668f, -1.1718279123306274f, 1.6178005933761597f, -1.7486406564712524f, -2.919844627380371f, 7.490941047668457f, -0.875328004360199f, -5.549253940582275f, -8.682398796081543f, 12.354720115661621f, -6.518106937408447f, 3.3926303386688232f, -7.628448486328125f, -0.5480865240097046f, 2.757507085800171f, -5.0333638191223145f, -12.717732429504395f, 9.747604370117188f, -14.137182235717773f, 13.830326080322266f, -8.072053909301758f, 9.366421699523926f, 4.771058082580566f, -3.8767879009246826f, 12.098913192749023f, -1.3983639478683472f, -2.516928195953369f, 17.085342407226562f, 0.700747013092041f, -4.593225479125977f, 3.2615785598754883f, 4.119733810424805f, 8.652438163757324f, 7.159962177276611f, -6.102161407470703f, -7.0815653800964355f, -4.8683037757873535f, -8.63952350616455f, -3.9259283542633057f, 9.596209526062012f, -9.913541793823242f, 3.975876569747925f, -3.891753911972046f, 5.650779724121094f, -6.815282344818115f, 2.731234550476074f, -0.2151940017938614f, -4.856814384460449f, 0.45669957995414734f, -7.445348262786865f, 9.566651344299316f, 1.5104864835739136f, 11.227133750915527f, -0.11537183076143265f, 4.145509719848633f, 4.6820244789123535f, 6.9148335456848145f, 4.9869384765625f, 6.348864555358887f, 0.12999588251113892f, -12.564311981201172f, -4.362220764160156f, -2.023157835006714f, -5.578927516937256f, 10.822314262390137f, -8.657614707946777f, 11.476557731628418f, -7.976850509643555f, 8.056036949157715f, 3.8919501304626465f, -3.0189290046691895f, 0.370013564825058f, 7.211360454559326f, 6.979368209838867f, -5.752582550048828f, -18.53067398071289f, -0.06240836903452873f, 6.615059852600098f, 14.322349548339844f, 7.855193614959717f, -2.3387138843536377f, -7.839303970336914f, -1.904661774635315f, 1.820509672164917f, 0.5016192197799683f, -14.7440767288208f, 13.704774856567383f, 11.45584774017334f, 10.099801063537598f, 0.022373056039214134f, -2.2562215328216553f, -3.3610339164733887f, -18.4713134765625f, -10.077775955200195f, 8.24351978302002f, -11.9453125f, 1.6951130628585815f, -6.876453399658203f, -9.18753433227539f, -3.261322498321533f, -5.389860153198242f, 1.553828239440918f, -3.9102296829223633f, 6.239824295043945f, 0.8332391381263733f, -0.4328247308731079f, -8.079927444458008f, 3.283322334289551f, 2.487851619720459f, 3.38568115234375f, 1.1550346612930298f, 5.795931339263916f, -3.827712059020996f, -17.76996612548828f, -1.3455630540847778f, -8.596356391906738f, -1.9443339109420776f, -3.347691535949707f, -3.8022572994232178f, -1.9706828594207764f, -8.254030227661133f, 4.786689758300781f, -9.022886276245117f, -10.170120239257812f, -7.9390668869018555f, -5.307765960693359f, -1.4348095655441284f, -4.2725114822387695f, 1.155957579612732f, 7.248950004577637f, 1.4605406522750854f, 2.4984130859375f, -4.926766395568848f, -1.360730767250061f, -3.5312368869781494f, -2.6760661602020264f, 13.249751091003418f, -2.3633241653442383f, 0.16462773084640503f, 2.9756968021392822f, -0.7048916220664978f, 3.0361123085021973f, 2.4167332649230957f, -18.724096298217773f, -2.668437957763672f, 4.492554664611816f, -1.228598952293396f, 4.189570426940918f, 5.219515323638916f, 1.4071485996246338f, -9.033429145812988f, 5.843366622924805f, 9.23009967803955f, -1.3791900873184204f, 2.5881869792938232f, 6.0501861572265625f, 10.04381275177002f, -6.157512187957764f, 7.24165678024292f, -4.1937384605407715f, -8.702505111694336f, 4.425203800201416f, -11.129894256591797f, -3.9875574111938477f, 10.64867877960205f, 5.056559085845947f, -10.747085571289062f, 1.264474630355835f, 7.634984016418457f, -16.58453369140625f, 8.581056594848633f, 6.325976848602295f, 2.503847360610962f, -1.0785939693450928f, 9.530458450317383f, -6.161625862121582f, -7.124795436859131f, 7.468658447265625f, -3.5646467208862305f, -2.393043041229248f, 10.85168743133545f, -1.2992841005325317f, 1.1869677305221558f, 12.415637969970703f, -0.384156733751297f, 1.6292704343795776f, -9.484331130981445f, -1.8839619159698486f, 7.0338287353515625f, -7.619279861450195f, 1.280126690864563f, -3.183976888656616f, -3.0420706272125244f, -3.4128353595733643f, -11.282638549804688f, -2.9777684211730957f, 0.18747295439243317f, -9.763144493103027f, 5.693328857421875f, 6.713829040527344f, 6.432366847991943f, 6.775195121765137f, 15.871235847473145f, -5.289862632751465f, 4.2354960441589355f, -2.9994354248046875f, -1.6106607913970947f, -8.912858009338379f, -9.295467376708984f, -8.503449440002441f, -4.607176303863525f, -1.5842245817184448f, 0.7097844481468201f, 10.443241119384766f, -0.1091175451874733f, -0.7141821384429932f, 0.6914375424385071f, -3.269829750061035f, -8.911444664001465f, -1.1965527534484863f, -1.024635910987854f, 3.5703232288360596f, -1.7879712581634521f, -3.1247658729553223f, -3.874403476715088f, -5.389334678649902f, 15.912910461425781f, -1.5096516609191895f, 10.253766059875488f, 5.733814716339111f, 0.046457380056381226f, -4.8697710037231445f, -6.196042537689209f, 26.0557861328125f, 6.26172399520874f, -6.333352088928223f, 3.3930180072784424f, -10.041016578674316f, -4.201577663421631f, -17.489336013793945f, -7.753904342651367f, -3.3036975860595703f, 9.259572982788086f, 5.622028350830078f, 7.418932914733887f, -4.166983127593994f, 3.4362385272979736f, 7.763801574707031f, 10.344937324523926f, 11.575509071350098f, -6.547769069671631f, 7.228475570678711f, 0.27201998233795166f, 11.231034278869629f, 15.090219497680664f, -3.0461761951446533f, 2.7282047271728516f, 0.22180375456809998f, 0.6711186170578003f, 7.804783821105957f, 1.4017285108566284f, 8.923149108886719f, -12.729116439819336f, 7.182803153991699f, 3.5145866870880127f, -2.1636273860931396f, -22.26779556274414f, -11.131207466125488f, -3.5905392169952393f, 25.306209564208984f, -4.522180557250977f, 8.663426399230957f, -2.190467357635498f, -9.864779472351074f, 0.2012891173362732f, -6.670775890350342f, -4.750314712524414f, 11.050165176391602f, -5.542118549346924f, 9.413408279418945f, 2.308258295059204f, 6.050876617431641f, 5.7319722175598145f, 3.2697877883911133f, -5.117337226867676f, 6.679595947265625f, 3.9800209999084473f, -13.86974048614502f, 4.446991920471191f, -8.73182201385498f, -4.240108966827393f, -3.3107078075408936f, 5.727823734283447f, -1.7871451377868652f, -1.1543947458267212f, 3.274351119995117f, 8.465099334716797f, 4.256385326385498f, 1.6312638521194458f, -0.4266124665737152f, -10.066319465637207f, -1.1442700624465942f, -13.979047775268555f, 9.38019847869873f, 10.103825569152832f, 1.44222891330719f, -12.20266056060791f, -4.161468505859375f, 5.828897476196289f, 3.763794422149658f, 3.17692494392395f, -7.616631507873535f, -4.522240161895752f, 1.3924592733383179f, 3.9834253787994385f, 7.975732326507568f, -5.482377529144287f, -10.70419979095459f, -1.8837058544158936f, -5.173501968383789f, -0.19477716088294983f, -3.4711830615997314f, 10.08415699005127f, -1.9378299713134766f, 5.39734411239624f, -4.766819953918457f, -7.339071750640869f, 4.702202796936035f, 6.930264949798584f, 14.5244779586792f, 1.373946189880371f, -7.17212438583374f, 4.260952949523926f, 13.407155990600586f, 1.6655051708221436f, 9.432223320007324f, 10.111971855163574f, -19.2701473236084f, 7.098327159881592f, -10.078179359436035f, 2.730682373046875f, -6.775233268737793f, 2.490149974822998f, 6.198461532592773f, 0.2401915341615677f, 13.640817642211914f, 6.729548454284668f, -5.323953151702881f, 7.341331958770752f, 0.6824727058410645f, -1.65504789352417f, -13.688631057739258f, 4.6387176513671875f, 3.3633487224578857f, -13.678863525390625f, 5.560359477996826f, -2.417947769165039f, 6.356764316558838f, 11.856796264648438f, -4.349381446838379f, -13.190492630004883f, 0.9404691457748413f, -0.07949790358543396f, 5.1438212394714355f, -5.610644817352295f, 9.887959480285645f, -10.704916000366211f, -6.3003129959106445f, 1.457470178604126f, -5.060678005218506f, 3.0186398029327393f, 0.4794176518917084f, 5.93607759475708f, 5.506703853607178f, -1.0873303413391113f, 2.6558244228363037f, 5.770946502685547f, -4.9762678146362305f, 12.409749031066895f, -7.4365234375f, 8.309518814086914f, 0.3051859140396118f, -6.369776248931885f, -6.938343048095703f, 14.529619216918945f, -8.19320011138916f, 4.215144157409668f, -8.835214614868164f, 2.212118148803711f, 7.021667003631592f, 1.2244818210601807f, -6.6155571937561035f, -5.983504295349121f, 5.949681758880615f, -9.470444679260254f, 1.7039216756820679f, 4.452876567840576f, 3.5223419666290283f, 1.386846661567688f, 12.392443656921387f, 3.4754412174224854f, -3.757160186767578f, 2.243192672729492f, 1.5498485565185547f, -5.969877243041992f, -9.53218936920166f, -0.9648003578186035f, -4.1782612800598145f, -8.63139533996582f, 2.888253688812256f, -4.9197845458984375f, -2.6188461780548096f, -0.34100860357284546f, 1.08323073387146f, 0.20715828239917755f, 0.17625738680362701f, 0.8104458451271057f, 6.235602378845215f, 1.1863518953323364f, 2.2748658657073975f, -2.9206295013427734f, -7.237782001495361f, 7.247722625732422f, -4.8302412033081055f, -16.587038040161133f, 3.3862411975860596f, 5.526927471160889f, -2.761680841445923f, -5.178590774536133f, 12.247193336486816f, -13.684733390808105f, -5.715342998504639f, -8.004195213317871f, 11.39901065826416f, -8.808146476745605f, -11.446958541870117f, -5.106409549713135f, -2.114779472351074f, 0.827870786190033f, 7.99212121963501f, 16.132234573364258f, -7.876288414001465f, 4.045842170715332f, 4.89345121383667f, 7.8902387619018555f, 2.6189887523651123f, 1.4187779426574707f, -4.95357608795166f, 1.9183154106140137f, -10.423624038696289f, 5.698985576629639f, -10.64306640625f, 8.366253852844238f, -6.523881435394287f, -12.701069831848145f, 18.55314064025879f, -17.26905059814453f, -2.0632481575012207f, 8.771729469299316f, 6.320028781890869f, 2.5256295204162598f, 14.239582061767578f, 3.1535520553588867f, 4.2085394859313965f, 0.1063476949930191f, 2.853101968765259f, 7.213879585266113f, 7.257656097412109f, -5.45920467376709f, -4.268126010894775f, -1.759271502494812f, -0.4200115203857422f, 3.3460514545440674f, 3.7233402729034424f, 1.9605106115341187f, 10.866938591003418f, -3.5412983894348145f, 0.06446467339992523f, -13.610902786254883f, -0.3217482566833496f, -13.58945083618164f, 7.48920202255249f, -4.73434591293335f, -4.7902374267578125f, 15.362975120544434f, -2.182408571243286f, 5.474664211273193f, -9.107425689697266f, 1.9057763814926147f, 5.691718101501465f, -5.837611675262451f, 0.9479477405548096f, 5.760697841644287f, 0.09852147847414017f, 4.3325629234313965f, -1.1939945220947266f, -9.364533424377441f, 5.033231735229492f, 4.058061122894287f, -13.785080909729004f, 1.9617301225662231f, -15.00662899017334f, 1.1908645629882812f, 1.5933691263198853f, -3.296618938446045f, 0.3798801302909851f, 3.9930577278137207f, 13.144612312316895f, -19.245630264282227f, 3.366441011428833f, -9.155226707458496f, -4.203514099121094f, -6.379355430603027f, 3.052100896835327f, 3.3825178146362305f, -5.03127384185791f, -20.389877319335938f, 15.295291900634766f, -17.634958267211914f, 7.198425769805908f, -26.755468368530273f, -5.668976783752441f, 9.652477264404297f, 1.5629017353057861f, -3.2165329456329346f, -2.528200626373291f, 8.87691593170166f, 1.8550244569778442f, 12.378725051879883f, 0.00554652651771903f, 5.4776787757873535f, -0.582221508026123f, -3.155843734741211f, 12.177905082702637f, -0.9750184416770935f, -0.16764011979103088f, -19.229124069213867f, 5.547354221343994f, 14.744590759277344f, 7.076948642730713f, 11.192045211791992f, 8.736725807189941f, -10.47435188293457f, -1.8428961038589478f, 8.822990417480469f, 10.079815864562988f, -9.724599838256836f, 4.329649925231934f, 22.17165756225586f, -12.209203720092773f, 6.37702751159668f, -15.575246810913086f, 4.703880786895752f, -1.969223976135254f, 0.45949697494506836f, 0.20915208756923676f, 11.847763061523438f, -3.744934558868408f, -13.487935066223145f, 3.874084234237671f, -4.578466415405273f, -7.086031436920166f, -6.233429908752441f, -6.785592555999756f, -3.3219637870788574f, 8.725717544555664f, 8.299102783203125f, 12.64601993560791f, -5.9381513595581055f, 0.5933879613876343f, 10.182705879211426f, 5.442933559417725f, 1.8527637720108032f, -4.759885787963867f, -4.977668285369873f, -7.128690719604492f, -2.2508225440979004f, -5.206679344177246f, 1.6984038352966309f, 3.4979476928710938f, -0.9469919204711914f, -5.347060680389404f, 2.9652979373931885f, -3.192228317260742f, 4.303493499755859f, 9.290297508239746f, 9.933444023132324f, -5.329427719116211f, 2.070354700088501f, 2.8292171955108643f, 13.59816837310791f, -2.8414688110351562f, 2.1585311889648438f, 5.236742973327637f, -2.824960947036743f, -18.918386459350586f, 0.8947755098342896f, -7.695527076721191f, 1.7885832786560059f, -18.260061264038086f, -15.04204273223877f, -1.8974013328552246f, -9.819128036499023f, 8.226827621459961f, -5.237205505371094f, 4.667030334472656f, -13.77105712890625f, 4.586790084838867f, -0.390918493270874f, -4.865528106689453f, -5.15276575088501f, 8.497437477111816f, 1.938669204711914f, -4.695268154144287f, 8.078354835510254f, 4.665389537811279f, -8.80441665649414f, 10.505725860595703f, -7.680267810821533f, -1.3688803911209106f, -3.3594307899475098f, 5.555975437164307f, 11.535223007202148f, -0.9209914803504944f, 2.747661590576172f, -14.198168754577637f, -2.046473503112793f, 9.80955982208252f, 0.09400451183319092f, 7.27423095703125f, 8.030008316040039f, -7.390738487243652f, 0.04207156226038933f, 6.497010231018066f, 2.9516983032226562f, -5.786115646362305f, 6.287818908691406f, 14.243786811828613f, -8.465189933776855f, -1.7876099348068237f, -5.9232072830200195f, 5.831533908843994f, -6.165143013000488f, 6.243556499481201f, -3.9629876613616943f, 10.030241966247559f, 13.016631126403809f, -6.402768611907959f, -3.428553819656372f, -2.3323075771331787f, 0.22513322532176971f, 9.516219139099121f, 8.901734352111816f, 1.7534918785095215f, 6.5073628425598145f, -11.850318908691406f, 22.981639862060547f, 0.9134898781776428f, -6.708924293518066f, 8.352168083190918f, 6.139276027679443f, -11.110394477844238f, -12.100716590881348f, 3.7225136756896973f, -6.486250400543213f, 4.79709529876709f, 3.2657382488250732f, -6.960175514221191f, 4.827823162078857f, 3.0818111896514893f, 4.233963966369629f, -1.300526738166809f, 8.41450023651123f, 27.55376625061035f, 6.323548793792725f, 1.2947889566421509f, -4.953489303588867f, 1.9106656312942505f, 1.7545992136001587f, 4.909358024597168f, -2.6168389320373535f, -13.989197731018066f, 2.96791672706604f, -15.628793716430664f, -4.646886348724365f, 2.3803651332855225f, -0.38021320104599f, 7.955515384674072f, 10.828896522521973f, -4.704763412475586f, -2.4767675399780273f, 4.345104694366455f, 9.674301147460938f, 14.051787376403809f, -1.9404326677322388f, -6.1034440994262695f, 4.96730375289917f, 8.356350898742676f, -12.492217063903809f, 5.052522659301758f, -11.288028717041016f, -18.85377311706543f, -2.1773197650909424f, 3.268491268157959f, -11.059211730957031f, 12.636406898498535f, 8.842144966125488f, 4.771931171417236f, 13.92164134979248f, -9.583444595336914f, 18.409446716308594f, -7.4875359535217285f, 24.48175621032715f, 5.571856498718262f, 3.763981819152832f, 2.6589486598968506f, -6.752713680267334f, -14.821791648864746f, -4.6671833992004395f, -7.933231830596924f, -5.372705459594727f, -16.37500762939453f, 8.258382797241211f, 4.597978115081787f, -4.5285491943359375f, -16.733186721801758f, -15.706796646118164f, -9.255148887634277f, 5.370093822479248f, -2.010138988494873f, 3.0411553382873535f, 5.09183406829834f, -0.09162905812263489f, 5.09684419631958f, 7.237029075622559f, -1.3704605102539062f, 15.3106107711792f, -9.188556671142578f, -2.035550117492676f, -11.181009292602539f, -3.858069658279419f, 0.42419710755348206f, 10.823795318603516f, 2.402636766433716f, -2.0442917346954346f, 2.90970778465271f, -10.479278564453125f, -5.484250068664551f, -5.404007434844971f, 5.897701263427734f, -10.198234558105469f, 3.919404983520508f, 11.954742431640625f, -3.368873119354248f, -2.416825532913208f, -5.485574245452881f, -1.0831294059753418f, 12.187384605407715f, 9.810623168945312f, 2.734238624572754f, -3.1333184242248535f, -5.630797386169434f, -3.6640121936798096f, -5.316211223602295f, -5.74959135055542f, -0.2944042682647705f, -23.43301010131836f, -3.242238998413086f, 3.7050588130950928f, 10.571874618530273f, 10.871262550354004f, 0.4957568347454071f, -13.565269470214844f, -0.34055426716804504f, -5.27890682220459f, -8.633820533752441f, -4.941458702087402f, -1.589221715927124f, 0.5624170899391174f, 5.252312183380127f, -6.480530261993408f, 1.477312445640564f, 3.0334291458129883f, 1.7544362545013428f, -1.811592698097229f, 2.166680097579956f, -2.746460199356079f, -14.217162132263184f, 0.9965202212333679f, -3.636343479156494f, -4.826642990112305f, -3.4294586181640625f, -13.18789291381836f, 1.1160309314727783f, 0.03090650774538517f, 4.610648155212402f, -6.316239356994629f, 5.343184947967529f, -3.5397722721099854f, 13.128379821777344f, 6.051750659942627f, 3.017526626586914f, -0.22869278490543365f, 1.0593830347061157f, -4.756152153015137f, -9.622599601745605f, -14.170649528503418f, 2.4119977951049805f, 3.543015241622925f, -6.777645111083984f, -2.0697014331817627f, -1.148790717124939f, -5.165157318115234f, -2.193115472793579f, 2.605295181274414f, -12.414362907409668f, -16.186542510986328f, -5.055665969848633f, 14.077547073364258f, -9.300932884216309f, 4.403715133666992f, -3.700418710708618f, 1.266947865486145f, 4.215131759643555f, 10.175348281860352f, -0.7335017919540405f, -6.666941165924072f, 1.7698924541473389f, 2.5551509857177734f, 14.33661937713623f, -6.243782043457031f, -7.089589595794678f, 5.264889240264893f, 2.0064051151275635f, -1.1940221786499023f, -0.8845052719116211f, 4.728024959564209f, 9.143659591674805f, -11.243470191955566f, -3.881427764892578f, 1.0975744724273682f, 5.020593166351318f, -2.509296178817749f, -0.7040889263153076f, -8.826937675476074f, 9.25462818145752f, 0.6523112058639526f, 8.413361549377441f, -9.956643104553223f, 4.320390701293945f, -7.327249050140381f, 0.7185436487197876f, 5.529834747314453f, 3.7425594329833984f, 11.977640151977539f, 0.3199857473373413f, -1.2970579862594604f, -0.6297903656959534f, -6.748528480529785f, -1.8058745861053467f, 7.02766227722168f, -14.052090644836426f, 3.328584909439087f, -11.310714721679688f, -8.255908966064453f, -1.6174887418746948f, 2.040614604949951f, -7.178774833679199f, -8.185322761535645f, -12.005358695983887f, -1.7305762767791748f, -15.079358100891113f, -8.002707481384277f, -13.309523582458496f, -4.7150774002075195f, -7.385843276977539f, 3.19026517868042f, -0.39262112975120544f, 10.768528938293457f, -12.257975578308105f, -20.101083755493164f, -10.958288192749023f, 7.338348388671875f, -5.042275905609131f, -4.23043966293335f, -4.301355361938477f, 6.800642967224121f, 0.3120475709438324f, 10.304023742675781f, -13.633195877075195f, -0.5820647478103638f, 5.642172813415527f, -7.806666851043701f, 4.420014381408691f, -11.09785270690918f, -0.6035222411155701f, -3.352290630340576f, -11.287544250488281f, -9.170281410217285f, 4.307720184326172f, 5.357173442840576f, 9.775383949279785f, -4.904134273529053f, 0.8738774657249451f, -1.9923650026321411f, -13.437588691711426f, 4.374331951141357f, -6.224425315856934f, -7.205249786376953f, -1.2282158136367798f, 16.407073974609375f, 14.745433807373047f, 10.053177833557129f, -5.026731014251709f, 13.399908065795898f, -4.700345039367676f, -2.810746908187866f, 4.868433952331543f, 14.18775463104248f, 0.07687458395957947f, -1.2327467203140259f, -1.3728224039077759f, 7.709836483001709f, 3.666210889816284f, 4.985335826873779f, -12.233044624328613f, -5.2307329177856445f, -5.411324977874756f, 5.240880012512207f, -2.543198347091675f, -21.135807037353516f, -11.415369987487793f, 5.937788486480713f, -0.01360476203262806f, -15.29384708404541f, -23.956432342529297f, 1.6906027793884277f, 3.3760499954223633f, 9.199305534362793f, 12.791433334350586f, 4.497912883758545f, 3.4790523052215576f, -2.8718466758728027f, 4.7231268882751465f, -12.663939476013184f, 8.505736351013184f, 0.8276928663253784f, -6.634997367858887f, 3.213438034057617f, -12.446330070495605f, -0.040080517530441284f, 8.142585754394531f, -5.221867084503174f, -7.002190113067627f, -0.7204623818397522f, -11.662397384643555f, -1.66923189163208f, 2.7600550651550293f, -7.987179279327393f, 1.4977517127990723f, -1.5806363821029663f, -3.7816100120544434f, 0.5635890960693359f, 1.5413968563079834f, -1.8732370138168335f, 0.9285376667976379f, -21.148822784423828f, 19.22353744506836f, -2.7432806491851807f, -14.096722602844238f, 8.600056648254395f, -8.63492488861084f, -0.5446416139602661f, -9.644774436950684f, 1.5598598718643188f, 9.11766242980957f, 11.372527122497559f, -12.296475410461426f, 7.532688617706299f, -8.911373138427734f, 6.434876441955566f, 3.617435932159424f, 4.387059688568115f, 12.811718940734863f, -9.318404197692871f, -0.4164440929889679f, -4.88645601272583f, -9.689740180969238f, -7.375895023345947f, 3.4132399559020996f, -5.05049991607666f, -4.028200626373291f, -8.181990623474121f, -7.417090892791748f, -6.058056831359863f, 4.370342254638672f, 4.167178630828857f, 10.502394676208496f, -8.585572242736816f, -7.519354820251465f, 11.942482948303223f, 13.272440910339355f, -2.757564067840576f, 1.3308485746383667f, -2.266592264175415f, -6.452311992645264f, -8.549099922180176f, -6.8195576667785645f, -6.657449722290039f, -11.0411376953125f, -21.244892120361328f, -9.757211685180664f, 3.444068670272827f, 5.706812858581543f, -1.7444701194763184f, 8.521875381469727f, -5.228854179382324f, -5.382219314575195f, -6.763889789581299f, -11.332088470458984f, -5.202573299407959f, 1.0299209356307983f, -11.423958778381348f, -1.0182582139968872f, 12.390052795410156f, -10.38305950164795f, -5.015763282775879f, 10.922207832336426f, -2.7742760181427f, 16.766313552856445f, -7.944525241851807f, -2.8002591133117676f, 10.874272346496582f, -9.857773780822754f, 2.1176915168762207f, -7.479413032531738f, -1.7775098085403442f, 8.738555908203125f, -1.012858271598816f, -0.524698793888092f, -2.0892815589904785f, 2.1344618797302246f, 2.1381497383117676f, -8.60706615447998f, -3.834965944290161f, 5.380415916442871f, -5.791232109069824f, 12.458983421325684f, -2.1414995193481445f, -6.775130271911621f, 6.628716468811035f, -6.176581382751465f, 1.7703351974487305f, 4.631689071655273f, 6.3223066329956055f, 2.441192388534546f, -5.126554012298584f, -1.0228726863861084f, 2.649749517440796f, 2.357867956161499f, -2.623882532119751f, 0.692119300365448f, 7.061679840087891f, -1.3346446752548218f, 9.300801277160645f, -2.3639140129089355f, -14.160957336425781f, 4.440097808837891f, 1.5377870798110962f, -2.905083179473877f, -14.448901176452637f, 3.674952983856201f, -3.8605237007141113f, 8.271306037902832f, 6.405794620513916f, -21.742176055908203f, -11.455822944641113f, 1.7873941659927368f, -11.738869667053223f, 8.434774398803711f, -20.15655517578125f, -1.367598056793213f, 7.51062536239624f, -7.572475910186768f, -0.1449132263660431f, 0.14903344213962555f, 1.539661169052124f, -8.20009994506836f, -7.516574382781982f, 6.837973117828369f, -5.781085968017578f, 1.294516921043396f, -9.226051330566406f, 1.2180476188659668f, -4.204068183898926f, 15.320548057556152f, -6.9747209548950195f, 10.799696922302246f, 9.24210262298584f, 9.05672550201416f, -7.968204498291016f, -4.376702308654785f, -2.1221911907196045f, 0.1324523389339447f, 4.775428771972656f, -0.22519434988498688f, -7.35526180267334f, 0.1219637468457222f, -11.087621688842773f, -7.007081985473633f, 5.306663990020752f, -16.567237854003906f, -9.150774002075195f, 19.48033332824707f, 9.253161430358887f, 8.751167297363281f, 6.719225883483887f, 5.08009672164917f, -6.268258571624756f, 0.8974515795707703f, -12.448920249938965f, 19.20334815979004f, -12.337990760803223f, -2.6060383319854736f, 17.232389450073242f, -5.7053303718566895f, -2.461069107055664f, -9.090248107910156f, -2.2077951431274414f, 5.545995235443115f, 2.147212028503418f, 8.779653549194336f, -1.8891314268112183f, 10.41473388671875f, 5.098225116729736f, 3.8328142166137695f, 4.6025872230529785f, -3.5405564308166504f, 4.929543972015381f, -15.190320014953613f, -4.475381851196289f, 4.857414722442627f, -0.2330942153930664f, -9.112611770629883f, -14.155166625976562f, -14.117636680603027f, 0.3170005977153778f, -6.148338794708252f, 2.6847426891326904f, 1.3375723361968994f, -10.719864845275879f, 19.82689666748047f, -11.773765563964844f, 1.4471533298492432f, 5.438664436340332f, 10.833401679992676f, 5.91309928894043f, -4.8504557609558105f, -0.8866681456565857f, 4.502740383148193f, 8.76336669921875f, -7.2705078125f, -2.602914810180664f, 0.4528222978115082f, 0.2632019817829132f, -3.9463186264038086f, -6.615810394287109f, -12.216948509216309f, 18.130096435546875f, -11.216374397277832f, -9.20461368560791f, 2.837886333465576f, -9.17361068725586f, -13.872835159301758f, 1.4739644527435303f, -4.641720294952393f, -6.107040882110596f, -0.7774397730827332f, 2.1887550354003906f, -3.053284168243408f, 6.492974758148193f, -8.238874435424805f, -17.8471622467041f, -5.672789096832275f, -2.3216474056243896f, -1.9701411724090576f, -7.030838966369629f, -2.107118844985962f, -0.4771590828895569f, -1.8420956134796143f, -14.038286209106445f, -4.026478290557861f, 3.295030355453491f, 4.460195064544678f, 6.017807960510254f, 7.253128528594971f, -7.15041971206665f, 0.515494167804718f, 4.030392646789551f, -3.3914072513580322f, 0.44524064660072327f, -0.19258594512939453f, -0.7080201506614685f, -2.50250244140625f, 2.334340810775757f, -11.007967948913574f, -12.750214576721191f, -7.300755977630615f, 14.395295143127441f, -3.3914880752563477f, 8.228107452392578f, 9.74307918548584f, 0.77601557970047f, 2.347871780395508f, 9.640531539916992f, -4.919435024261475f, -7.991837024688721f, 0.03702786564826965f, 0.6593180894851685f, -8.348962783813477f, 4.535181999206543f, 3.7413909435272217f, -4.0721588134765625f, 7.057863712310791f, -1.0119569301605225f, -5.5510711669921875f, -11.117044448852539f, 5.491680145263672f, 12.420897483825684f, 12.342089653015137f, -13.135498046875f, 0.531191349029541f, -3.1611971855163574f, 2.2477285861968994f, -4.986516952514648f, 7.761373996734619f, 2.0133655071258545f, 9.318339347839355f, 8.271308898925781f, 2.2258410453796387f, 7.38371467590332f, 1.7697837352752686f, 7.604837894439697f, -5.065467357635498f, 15.182759284973145f, 10.54971694946289f, 14.205101013183594f, -15.358622550964355f, -7.208074569702148f, -1.3423936367034912f, 4.829193115234375f, -6.763162612915039f, -6.093154430389404f, -3.8595669269561768f, 10.870450019836426f, 5.670471668243408f, -9.437480926513672f, 2.4107041358947754f, -6.857131004333496f, 2.4801251888275146f, -5.085203170776367f, 15.741228103637695f, -18.639524459838867f, 0.10690624266862869f, 10.266143798828125f, 0.7210645079612732f, 2.90657114982605f, 4.488924980163574f, 11.784412384033203f, 10.836816787719727f, 11.812047004699707f, 0.9760425090789795f, 6.398825645446777f, -11.69657039642334f, 9.617613792419434f, -2.4754092693328857f, 2.3830978870391846f, 4.122246742248535f, -6.237287521362305f, 6.2666802406311035f, -7.420814514160156f, -4.996058940887451f, 6.019990921020508f, 8.033538818359375f, -3.3236873149871826f, -10.9536714553833f, 6.0252366065979f, -10.10529613494873f, 10.607632637023926f, -1.0954471826553345f, -7.890513896942139f, -6.947789669036865f, 18.596343994140625f, -10.062642097473145f, 9.224631309509277f, 12.48892593383789f, 11.919086456298828f, 10.027180671691895f, 8.543699264526367f, -8.874430656433105f, -2.0335042476654053f, -14.272953033447266f, 3.9183735847473145f, -8.344069480895996f, -11.0534086227417f, 2.4750025272369385f, -0.029886547476053238f, 5.07313871383667f, -26.66839599609375f, -8.733036041259766f, -4.485158920288086f, 9.778528213500977f, 5.770241737365723f, 12.092632293701172f, 1.5609368085861206f, 11.50508975982666f, -0.6498426795005798f, -9.589741706848145f, -27.592344284057617f, -0.43285009264945984f, 11.695541381835938f, -6.151013374328613f, 5.020718097686768f, -12.249852180480957f, -2.1764540672302246f, 6.811885833740234f, 16.927019119262695f, -10.076769828796387f, -17.306066513061523f, 4.369997978210449f, 1.4093303680419922f, -2.871824026107788f, 2.7676126956939697f, -6.874851226806641f, -2.597337484359741f, -9.963288307189941f, 10.695664405822754f, 5.037891387939453f, -6.623497009277344f, 5.716047286987305f, 4.312154769897461f, 4.035192489624023f, 2.1116819381713867f, 3.357062578201294f, 1.416142225265503f, 3.6634714603424072f, -4.913036823272705f, -5.307648181915283f, 11.786884307861328f, -1.3370147943496704f, 10.858242988586426f, 8.861958503723145f, 6.27660608291626f, 12.768265724182129f, 12.327997207641602f, -8.675048828125f, -3.4217770099639893f, 7.692430019378662f, 1.7838584184646606f, 12.854020118713379f, -11.746981620788574f, -9.584903717041016f, 8.365598678588867f, 12.973992347717285f, 5.4280571937561035f, -9.301798820495605f, 11.192365646362305f, -2.602813243865967f, 6.336965084075928f, 0.5768340229988098f, -10.169051170349121f, -8.006020545959473f, 13.611660957336426f, -5.536235332489014f, 3.161245346069336f, -9.059117317199707f, -2.214181423187256f, -5.4537811279296875f, 6.635800838470459f, -12.098894119262695f, 11.331655502319336f, 13.68090534210205f, 3.1792664527893066f, 1.3622552156448364f, 2.2142395973205566f, 0.2345779687166214f, 0.5814613103866577f, 4.289124011993408f, 4.538444995880127f, -7.362942695617676f, 2.874087333679199f, 3.61981201171875f, -5.420032501220703f, 8.720592498779297f, 4.290198802947998f, 12.945777893066406f, 5.480059623718262f, -7.588292598724365f, 5.661287784576416f, -3.8283472061157227f, -8.462838172912598f, 7.285812854766846f, -10.747210502624512f, -3.003866672515869f, 7.091235637664795f, -1.580762505531311f, 8.499457359313965f, 0.9311726689338684f, 2.294097661972046f, 7.290738582611084f, -4.4237260818481445f, 13.710098266601562f, -7.216895580291748f, 2.5698392391204834f, -4.586067199707031f, -5.298479080200195f, -3.6828322410583496f, -1.1227331161499023f, 6.324790954589844f, 0.32012125849723816f, -0.4937366843223572f, -1.9536423683166504f, -9.237577438354492f, 1.0877151489257812f, 12.129581451416016f, 4.084137916564941f, -2.9383089542388916f, -5.019631862640381f, 0.6925498247146606f, 6.584831714630127f, 0.4018475115299225f, 0.21690116822719574f, 1.1919173002243042f, 2.2553939819335938f, -1.0278080701828003f, -13.67727279663086f, -1.5738617181777954f, 2.352642297744751f, 3.2829935550689697f, 0.22515998780727386f, -5.764885902404785f, -3.821402072906494f, 7.599716663360596f, 4.4758172035217285f, -18.555747985839844f, 10.015554428100586f, -5.277355194091797f, 4.857947826385498f, -4.972664833068848f, -7.69036865234375f, 0.8392868041992188f, -26.074081420898438f, -8.01816463470459f, 17.92840576171875f, 4.751870632171631f, 1.765329360961914f, -4.687397003173828f, 8.163311958312988f, 4.182957172393799f, 8.034849166870117f, 1.3160547018051147f, -0.5854548215866089f, -8.705289840698242f, -0.8203011155128479f, -3.8902640342712402f, -3.927004814147949f, 10.755240440368652f, -2.724745035171509f, -5.045247554779053f, -13.067852973937988f, 15.737813949584961f, 1.6259382963180542f, 0.018999869003891945f, 1.839993953704834f, 2.3978917598724365f, 1.3756414651870728f, 10.791954040527344f, -4.205836772918701f, 7.3124308586120605f, 3.0303399562835693f, 18.43343734741211f, 2.922717571258545f, 7.7857747077941895f, 9.202900886535645f, -11.383536338806152f, 4.063195705413818f, 4.0933308601379395f, 9.472505569458008f, 0.6550918817520142f, 1.219048261642456f, -3.383958101272583f, -13.222259521484375f, 6.909287452697754f, -11.573260307312012f, 4.1832146644592285f, 6.807440280914307f, -12.13596248626709f, -16.246288299560547f, -11.497807502746582f, 8.08572769165039f, -12.573715209960938f, -4.854974746704102f, -2.2262701988220215f, 0.8039400577545166f, 11.886690139770508f, -15.268168449401855f, -8.579140663146973f, -2.6850054264068604f, 5.244555950164795f, -6.70413064956665f, -3.2002172470092773f, 6.914629936218262f, 0.3705389201641083f, 14.895472526550293f, -6.253382205963135f, -6.998981475830078f, 4.563458442687988f, 5.477730751037598f, -9.057239532470703f, 9.500064849853516f, 4.373735427856445f, -0.7703867554664612f, 13.951043128967285f, -5.243332862854004f, 2.8435473442077637f, -6.590154647827148f, 4.319807529449463f, -2.5442638397216797f, -13.913573265075684f, -10.475417137145996f, -2.8347291946411133f, 8.493965148925781f, -10.500249862670898f, -5.360538005828857f, -4.234364986419678f, 0.10634779930114746f, -1.185592532157898f, -9.338807106018066f, 2.1675655841827393f, 3.200042486190796f, 4.2734856605529785f, 5.105844020843506f, -7.451834201812744f, -11.285500526428223f, -2.5912656784057617f, -4.90095853805542f, -7.117650508880615f, 5.817300796508789f, -2.694540023803711f, -6.723092555999756f, 3.3572030067443848f, 9.950608253479004f, -2.062587261199951f, -2.237545967102051f, 5.24613094329834f, 18.30078887939453f, -7.8479437828063965f, -2.280181407928467f, 8.82515811920166f, 0.1949726939201355f, 4.303898811340332f, -3.4921815395355225f, 16.898700714111328f, 11.57261848449707f, -2.4016921520233154f, -3.2015914916992188f, 10.793314933776855f, -6.424396514892578f, 1.5789941549301147f, 6.381648540496826f, -21.38960838317871f, 0.6480197906494141f, -13.997282028198242f, 0.6987282037734985f, -7.389518737792969f, 10.640852928161621f, -1.155595064163208f, -2.187462091445923f, -2.0233631134033203f, -2.092860698699951f, 8.194184303283691f, 12.357358932495117f, 12.201729774475098f, 5.951974868774414f, 1.862755537033081f, -4.316686630249023f, 6.706587791442871f, 5.658695220947266f, -3.7403807640075684f, 4.848812580108643f, -1.1044867038726807f, 4.073699474334717f, -6.804827690124512f, -2.227973461151123f, -14.886821746826172f, 4.097927093505859f, -5.149636745452881f, 14.20467472076416f, 10.393085479736328f, 16.598787307739258f, 20.412975311279297f, 0.4636867642402649f, -12.83670711517334f, 5.011075496673584f, 3.2573134899139404f, 2.541311502456665f, -3.704630136489868f, -7.621809482574463f, 2.2266361713409424f, -6.034920692443848f, -4.907690525054932f, 3.311673164367676f, -11.02491569519043f, 6.5720438957214355f, -13.933245658874512f, 3.847233295440674f, 12.274158477783203f, 0.9272652864456177f, -1.7610269784927368f, 0.4040965139865875f, 0.680636465549469f, -15.860685348510742f, 1.9717633724212646f, -9.28317928314209f, -2.6344847679138184f, 1.870112657546997f, -15.349178314208984f, -5.096085548400879f, -1.4747427701950073f, -8.14248275756836f, 2.2569987773895264f, -15.152320861816406f, -7.701322078704834f, 10.674273490905762f, -1.4894379377365112f, -4.1722493171691895f, -20.385786056518555f, -7.458427429199219f, 0.15474386513233185f, 6.180734634399414f, 9.458477973937988f, 3.9757118225097656f, 3.1847710609436035f, 2.5685551166534424f, -1.0029635429382324f, -0.9310358762741089f, -5.389017581939697f, 13.258286476135254f, 4.855055809020996f, 11.788922309875488f, 2.5168514251708984f, -8.065077781677246f, 5.255640983581543f, 12.24008846282959f, -13.184330940246582f, -0.7134942412376404f, -8.317489624023438f, 11.638908386230469f, -3.975881814956665f, 0.43127647042274475f, -8.266470909118652f, -12.825586318969727f, 13.573546409606934f, 8.546313285827637f, 4.274265289306641f, 3.6718971729278564f, 10.517722129821777f, 0.27425989508628845f, 8.075490951538086f, -2.1264874935150146f, 0.6136637926101685f, -5.506547451019287f, 15.231038093566895f, 0.46134939789772034f, 0.17480090260505676f, 2.040527582168579f, 3.2183902263641357f, -6.139379501342773f, -9.661615371704102f, -10.546405792236328f, -3.0357425212860107f, -3.1846868991851807f, -4.761189937591553f, -3.7215702533721924f, 3.0658512115478516f, -11.569316864013672f, -0.43601199984550476f, 2.8654866218566895f, -2.9958226680755615f, 2.4096181392669678f, -10.564579963684082f, -13.85832405090332f, -6.305012226104736f, 7.627152919769287f, 2.5408518314361572f, -10.277200698852539f, 1.9411969184875488f, -4.243484973907471f, 1.4209707975387573f, 3.253671169281006f, 1.769997000694275f, -9.947007179260254f, 3.4785919189453125f, 3.4994893074035645f, 14.112330436706543f, 17.531051635742188f, 3.3806848526000977f, 2.1774063110351562f, -14.381307601928711f, 6.56845235824585f, -11.52322769165039f, 0.6346222162246704f, 7.065883636474609f, -3.087952136993408f, 6.3684282302856445f, -6.367730140686035f, 6.013090133666992f, 4.485490322113037f, 3.8508636951446533f, -11.656087875366211f, 0.451434463262558f, 7.0519280433654785f, 6.291365623474121f, 10.597298622131348f, 7.171876430511475f, 1.4291812181472778f, -8.897068977355957f, 0.9057914614677429f, 1.817928671836853f, -3.542912006378174f, 15.661563873291016f, -3.5200037956237793f, -1.275672435760498f, 11.505273818969727f, -5.713345527648926f, -13.151139259338379f, 1.984048843383789f, 24.558940887451172f, -0.4125036895275116f, 7.0134735107421875f, 14.921035766601562f, 3.6945953369140625f, 15.289679527282715f, -12.831208229064941f, -7.242607593536377f, -14.355047225952148f, -3.520359992980957f, 17.282093048095703f, -1.093096137046814f, -4.698165416717529f, 4.53480863571167f, -2.2949156761169434f, 2.932422399520874f, 18.38007926940918f, 10.144601821899414f, 1.8887478113174438f, 17.05137825012207f, 2.2348875999450684f, 2.445681571960449f, -16.11703109741211f, 3.6988015174865723f, 28.14395523071289f, -9.436356544494629f, -4.76363468170166f, -9.584181785583496f, -0.4704954922199249f, 1.5917537212371826f, -21.702960968017578f, 6.3507513999938965f, -3.284785270690918f, 7.535861968994141f, 11.226126670837402f, 9.685650825500488f, 0.8183719515800476f, -15.07618236541748f, 6.108888149261475f, 16.537776947021484f, 12.913955688476562f, -4.378338813781738f, 7.780423641204834f, 7.1294660568237305f, 1.4495526552200317f, 0.26958322525024414f, 4.329139709472656f, 6.674724102020264f, -6.520862579345703f, -2.903130531311035f, -17.643451690673828f, 7.825291633605957f, -4.506794452667236f, 0.16557633876800537f, 7.2597246170043945f, 8.140376091003418f, -9.9315185546875f, -6.21274471282959f, -3.006944179534912f, 4.115133285522461f, 1.6233476400375366f, -8.518281936645508f, 1.5647332668304443f, -7.484006881713867f, -4.436017036437988f, 9.502815246582031f, -7.330345630645752f, 7.664921760559082f, -4.133516788482666f, 3.0703797340393066f, 7.408110618591309f, -1.3500370979309082f, -3.113905191421509f, -1.6690822839736938f, 9.191621780395508f, 0.6589356064796448f, -7.368348121643066f, 11.585264205932617f, 2.6871490478515625f, 5.345765113830566f, 1.0851669311523438f, -4.77912712097168f, -9.00320816040039f, -8.105009078979492f, 3.760385274887085f, 8.186501502990723f, 13.72744369506836f, 4.232875823974609f, 4.095627307891846f, 1.6133935451507568f, 14.020605087280273f, -17.89886474609375f, 10.849470138549805f, -4.106210708618164f, -1.5198999643325806f, -4.078025817871094f, 3.1078972816467285f, 6.467252254486084f, 8.850523948669434f, -1.770440936088562f, -3.601609706878662f, 12.48472785949707f, 1.1591122150421143f, 3.630988597869873f, 0.35214078426361084f, 9.08293628692627f, -9.61496353149414f, -3.6545019149780273f, 4.853860378265381f, -21.803064346313477f, 0.9578412175178528f, -5.081964492797852f, -3.8409125804901123f, 13.125861167907715f, 8.514156341552734f, 2.2802765369415283f, -1.5544071197509766f, 7.989774227142334f, -0.9534971117973328f, -20.655189514160156f, 5.728123664855957f, -4.578756809234619f, -9.761033058166504f, -2.411184072494507f, -1.53468918800354f, 1.7299505472183228f, 4.49857759475708f, -5.621171951293945f, -15.699580192565918f, -1.6200847625732422f, 0.2511891722679138f, 12.249534606933594f, -6.566256523132324f, -14.719828605651855f, -11.666041374206543f, 1.057192087173462f, -1.8811215162277222f, -4.319576263427734f, 9.619815826416016f, -6.46962833404541f, -3.5396673679351807f, -4.881474494934082f, -0.6955579519271851f, 3.299593687057495f, 11.470381736755371f, 1.12203049659729f, -0.921830952167511f, 5.743063449859619f, 6.646578311920166f, 1.6719251871109009f, -11.525588989257812f, -12.89569091796875f, 7.118905544281006f, -14.61849594116211f, -9.530622482299805f, -6.113729476928711f, 0.5888143181800842f, -17.381607055664062f, 15.451555252075195f, -6.282611846923828f, -12.82925033569336f, 1.7211908102035522f, -12.8479585647583f, 3.2159461975097656f, 2.13104510307312f, 0.8415284752845764f, -4.35218620300293f, -6.643268585205078f, -3.2159981727600098f, -1.1216778755187988f, -9.839357376098633f, -6.70488977432251f, -4.389339447021484f, -3.2769744396209717f, -5.768002510070801f, -8.732684135437012f, -5.264852046966553f, 6.483272552490234f, 9.102012634277344f, -14.003499984741211f, 0.17809408903121948f, 15.125717163085938f, -12.025708198547363f, -6.3086838722229f, 14.6791353225708f, 9.44178581237793f, -8.969319343566895f, -5.2899250984191895f, -9.268863677978516f, -6.997377872467041f, 9.275823593139648f, 1.1657955646514893f, -7.220368385314941f, 10.789793968200684f, 10.304367065429688f, -4.045272350311279f, -6.108043193817139f, -2.014784812927246f, -11.121515274047852f, -18.38974380493164f, -8.347667694091797f, 5.974706649780273f, 10.243739128112793f, 5.663839340209961f, 8.547972679138184f, -7.52782678604126f, 5.363646507263184f, 15.58993148803711f, 7.026348114013672f, 2.677098512649536f, 18.409914016723633f, -10.548135757446289f, 3.4355015754699707f, 1.4359816312789917f, -2.546203374862671f, -6.266178131103516f, 11.344420433044434f, -19.2717227935791f, -6.462072372436523f, 11.376580238342285f, 3.8701956272125244f, -8.504956245422363f, -7.087599277496338f, -11.178908348083496f, 0.2572647035121918f, 4.125010967254639f, 9.298619270324707f, -8.014323234558105f, 12.535018920898438f, 6.390746116638184f, -6.37876558303833f, 6.679197788238525f, -9.323160171508789f, -5.269445419311523f, 2.596299886703491f, 0.11649034917354584f, 17.804271697998047f, 4.594757556915283f, 6.764013767242432f, 0.6217585206031799f, -7.7316412925720215f, 8.208429336547852f, -7.3180928230285645f, -9.159605979919434f, 8.136429786682129f, -15.975767135620117f, -5.516753673553467f, -7.645376205444336f, -6.816018104553223f, -1.813738226890564f, -0.754987359046936f, 10.540684700012207f, 6.572773456573486f, 4.151686191558838f, 5.905804634094238f, -0.29068470001220703f, -5.357999324798584f, -5.847013473510742f, -4.441774845123291f, 1.8924025297164917f, -1.1913574934005737f, 14.646678924560547f, -13.63921070098877f, 1.4032258987426758f, -2.480900526046753f, 0.2637772262096405f, -7.374143600463867f, 5.216763019561768f, -22.21915626525879f, -1.6103098392486572f, 9.66075325012207f, 7.245777606964111f, -0.4308447539806366f, 3.232591152191162f, 7.705619812011719f, -3.5778160095214844f, -4.7608866691589355f, -3.9943084716796875f, -2.1508443355560303f, 7.583977222442627f, -11.541088104248047f, -9.87782096862793f, -0.3558848798274994f, -0.8333123326301575f, 0.36477258801460266f, -6.956092834472656f, -0.47873327136039734f, -7.288418769836426f, 1.8383797407150269f, -6.37021541595459f, -1.270717740058899f, -5.504342555999756f, -7.638356685638428f, 3.2992103099823f, 3.3208279609680176f, -7.895376682281494f, -0.4653431177139282f, -5.259646415710449f, 0.3640040159225464f, -4.201903820037842f, 7.085029125213623f, 0.7169751524925232f, 1.866424798965454f, 2.98149037361145f, 2.421513319015503f, -4.835546493530273f, -3.400444746017456f, -13.251764297485352f, -7.206017971038818f, 5.5685553550720215f, 8.815764427185059f, 1.3367613554000854f, -12.028512001037598f, -4.593379974365234f, -5.485569000244141f, 7.575770378112793f, 6.186659336090088f, 4.837860107421875f, 4.19438362121582f, -2.0015146732330322f, -8.463085174560547f, 4.390081882476807f, -0.967718243598938f, -6.633697032928467f, 16.146509170532227f, -12.346258163452148f, 5.003246307373047f, 5.257402420043945f, 5.420393943786621f, -4.056494235992432f, -22.63052749633789f, -3.4521031379699707f, 0.5595429539680481f, -0.6451472043991089f, 7.716019630432129f, -3.91341233253479f, 3.537018299102783f, 0.5044690370559692f, -6.045408725738525f, -3.7082343101501465f, 8.852910995483398f, -3.696688175201416f, -2.3492298126220703f, -9.58127212524414f, -12.638714790344238f, 1.8595764636993408f, -4.140509128570557f, -7.7006120681762695f, 1.3192996978759766f, -9.62076473236084f, 9.613175392150879f, 16.121721267700195f, 7.089335918426514f, -11.461114883422852f, -9.27320384979248f, 4.465617656707764f, 1.9801151752471924f, 11.973653793334961f, -9.35599136352539f, 5.719399929046631f, -7.321558475494385f, 17.419307708740234f, -10.423422813415527f, 4.022407531738281f, -1.3151211738586426f, -5.239253044128418f, 0.22252266108989716f, -23.80808448791504f, 11.66386890411377f, -15.182354927062988f, 2.0668842792510986f, -3.095346450805664f, -10.253707885742188f, 4.4772186279296875f, -16.292877197265625f, 2.14457106590271f, 7.256747245788574f, 0.5063409805297852f, -1.0078110694885254f, -17.32522201538086f, -2.448883295059204f, -7.290302753448486f, -10.84791374206543f, 4.010873317718506f, 0.7512247562408447f, -5.4202561378479f, 2.328165292739868f, -6.0993428230285645f, -12.868612289428711f, 4.567487716674805f, -10.598518371582031f, -11.058136940002441f, -1.659664273262024f, 10.452244758605957f, 18.823354721069336f, -8.08329963684082f, 4.401593208312988f, 2.138773202896118f, -13.085124969482422f, 2.258606195449829f, -2.7691009044647217f, 5.584531784057617f, 10.423367500305176f, 2.846891164779663f, -4.120229721069336f, 1.5436748266220093f, -13.3086519241333f, -8.132325172424316f, 13.937261581420898f, -14.859661102294922f, -10.103026390075684f, 10.011719703674316f, -0.9183381795883179f, 9.600658416748047f, -9.821544647216797f, -9.450371742248535f, -2.242305278778076f, -6.559144020080566f, -4.224136829376221f, -11.820085525512695f, -6.345705986022949f, 8.94025993347168f, 4.509256839752197f, -2.219336748123169f, -0.8802061080932617f, -14.329898834228516f, 7.088708877563477f, -9.526060104370117f, 8.278675079345703f, 2.4871132373809814f, -9.620868682861328f, 9.28262710571289f, 14.8040132522583f, -19.852964401245117f, 4.130612850189209f, 5.440346717834473f, 7.126719951629639f, 0.05489916726946831f, -1.4817479848861694f, 6.829895973205566f, 12.818050384521484f, 9.886153221130371f, 1.254286766052246f, -8.354349136352539f, -11.958121299743652f, 9.16726016998291f, 4.949143886566162f, -7.31182861328125f, 18.494693756103516f, -9.817893028259277f, -0.3160404562950134f, -7.171751022338867f, 15.233163833618164f, -2.146108388900757f, 5.443892478942871f, 4.517480850219727f, 14.043611526489258f, 10.903755187988281f, -7.276175022125244f, 13.366379737854004f, -8.15054702758789f, 0.6368334293365479f, 9.139420509338379f, -6.253448963165283f, 7.538908958435059f, -4.911654949188232f, 1.2663601636886597f, 5.10722017288208f, -10.726715087890625f, -9.355097770690918f, -0.3182983100414276f, -0.6880562901496887f, 1.4052690267562866f, 8.107300758361816f, -17.92100715637207f, -4.534452438354492f, 0.5386764407157898f, 12.413888931274414f, -7.201480388641357f, -13.179725646972656f, -8.37654972076416f, -0.3154694736003876f, 3.091693878173828f, -1.0480709075927734f, 3.751927614212036f, 5.383627414703369f, 6.276157855987549f, 5.926423072814941f, -12.69626235961914f, 0.419791042804718f, 5.777501583099365f, -2.8398115634918213f, -4.607639789581299f, 6.945610523223877f, -3.02091383934021f, -9.469738006591797f, -7.620004177093506f, -2.214186191558838f, -10.362244606018066f, -3.8059444427490234f, -8.182043075561523f, 12.854665756225586f, -9.528903007507324f, 0.49294862151145935f, -8.579874038696289f, 2.118196725845337f, 7.62219762802124f, -5.029037952423096f, 12.290265083312988f, 0.5394968390464783f, 0.8183786869049072f, 1.3815820217132568f, -8.656240463256836f, -10.229368209838867f, 5.616023540496826f, 1.3402541875839233f, 0.14747254550457f, 5.180647373199463f, -5.916377544403076f, 11.535149574279785f, 6.629460334777832f, -2.8352251052856445f, 2.0122580528259277f, -0.12555190920829773f, 0.09699758142232895f, -3.735365867614746f, 6.413780212402344f, -9.827960968017578f, 4.477472305297852f, 3.9674675464630127f, 13.4703950881958f, -10.233613014221191f, 6.505621433258057f, -0.8457767963409424f, 2.9881680011749268f, -1.2331901788711548f, 0.2708655893802643f, -20.598543167114258f, -3.6357667446136475f, -10.855840682983398f, 1.586439847946167f, 10.175971031188965f, -1.9043898582458496f, 2.3536105155944824f, 4.586009502410889f, -16.701967239379883f, -5.804449558258057f, -2.78130841255188f, -12.884259223937988f, 4.541162967681885f, -2.2364847660064697f, 7.2929511070251465f, -20.930879592895508f, -1.1628355979919434f, -0.6431397795677185f, -4.187624454498291f, 9.130221366882324f, 5.2739481925964355f, -5.0708441734313965f, 0.17360350489616394f, 0.5822219252586365f, -0.22339344024658203f, -2.761117458343506f, -10.778762817382812f, 5.385908603668213f, -7.857998371124268f, 2.606891393661499f, 18.175046920776367f, -2.693230390548706f, 10.186522483825684f, -10.02297306060791f, -2.4003751277923584f, 4.620128631591797f, 11.451003074645996f, -9.571845054626465f, -7.237552165985107f, 16.154325485229492f, -19.152278900146484f, 0.16639988124370575f, -1.1797699928283691f, -3.471789836883545f, 7.653792858123779f, 3.1148579120635986f, -17.236486434936523f, 7.4174604415893555f, 5.9619293212890625f, 3.140089511871338f, -1.4843370914459229f, -11.55695629119873f, 4.42191219329834f, -1.2158066034317017f, 1.4187449216842651f, -4.891499042510986f, -0.7901540398597717f, 3.7802929878234863f, -5.735926151275635f, -6.485677242279053f, -1.5401825904846191f, 0.4863590598106384f, 7.93129301071167f, -1.8886117935180664f, 13.101932525634766f, -9.918781280517578f, 7.965061664581299f, -7.907785892486572f, -1.5377079248428345f, 13.429678916931152f, -9.150714874267578f, 11.128018379211426f, 14.368767738342285f, 3.5130884647369385f, -20.24823570251465f, 5.9857892990112305f, -16.631755828857422f, 6.948392391204834f, -0.7420387864112854f, 13.481123924255371f, -6.017175674438477f, -6.115068435668945f, -3.0654659271240234f, 13.489151000976562f, -2.2497708797454834f, 15.904927253723145f, -5.288370132446289f, -2.09615421295166f, -1.6209585666656494f, -10.90034008026123f, -2.202252149581909f, -7.51241397857666f, 1.12482488155365f, -3.7093164920806885f, 5.637340068817139f, 3.773543357849121f, 1.558279275894165f, 0.7900522947311401f, 11.120495796203613f, -4.0744709968566895f, 10.195045471191406f, -1.018953561782837f, 15.167667388916016f, 2.601929187774658f, 0.008113066665828228f, -9.717181205749512f, 6.3429789543151855f, 21.926603317260742f, -9.98530387878418f, 14.128251075744629f, 6.466258525848389f, -6.853448390960693f, -4.719650745391846f, -11.167947769165039f, -3.2867848873138428f, -12.95023250579834f, -5.806682109832764f, 11.830602645874023f, -5.1555986404418945f, -6.032625198364258f, -7.19350528717041f, -0.5742126703262329f, -20.182292938232422f, -2.5560240745544434f, 8.388191223144531f, 3.965162754058838f, -2.2216339111328125f, -11.404376983642578f, 15.09308910369873f, 8.153555870056152f, -3.3941447734832764f, -5.3842549324035645f, 17.98311996459961f, -0.8321418166160583f, -13.44106388092041f, -20.923404693603516f, -7.57475471496582f, -3.0816946029663086f, 14.614544868469238f, 3.570502519607544f, 4.2990217208862305f, 0.1919950395822525f, 1.0185343027114868f, -8.14962387084961f, 10.906466484069824f, 15.584993362426758f, -0.4128416180610657f, -6.817087173461914f, 17.120927810668945f, 2.2402291297912598f, -1.9622392654418945f, -3.413076162338257f, 6.840425968170166f, -5.948558330535889f, 6.701462745666504f, 0.8215048909187317f, -1.0856260061264038f, -3.9645702838897705f, 1.4618407487869263f, -1.7632997035980225f, 0.5018937587738037f, -2.7499217987060547f, 4.106265068054199f, -12.30443000793457f, 6.145236492156982f, 4.5076003074646f, 3.7207319736480713f, 9.945277214050293f, 1.5612727403640747f, -25.549476623535156f, -4.634391784667969f, -9.542594909667969f, 10.549148559570312f, 2.471867322921753f, 1.1155062913894653f, -2.9077959060668945f, 7.583897590637207f, -2.4130048751831055f, 8.161357879638672f, -6.786379337310791f, 15.053579330444336f, 2.3771886825561523f, 8.431512832641602f, -10.34504508972168f, 3.9834461212158203f, 0.3579562306404114f, -14.247590065002441f, -11.214945793151855f, -6.256213188171387f, 8.011175155639648f, 9.249755859375f, -11.789390563964844f, -14.64880657196045f, -0.27090367674827576f, -0.2744891941547394f, 11.454398155212402f, -14.925914764404297f, -7.099286079406738f, -3.3692843914031982f, 6.4099931716918945f, 6.6391777992248535f, 10.544389724731445f, 3.6568782329559326f, 6.840872764587402f, 12.177023887634277f, 3.4128663539886475f, -3.8373212814331055f, -0.12041311711072922f, -13.457633972167969f, 18.881927490234375f, 8.021130561828613f, -7.102904796600342f, 7.734999179840088f, 3.826378107070923f, -0.7286080718040466f, -6.966921329498291f, -9.949929237365723f, -6.5738348960876465f, -6.720236778259277f, -2.4046998023986816f, -7.772119998931885f, -3.3762123584747314f, 16.03032875061035f, -10.771323204040527f, -0.8427897095680237f, 2.035958766937256f, -9.551191329956055f, 13.150819778442383f, -8.132336616516113f, -0.9750276803970337f, 2.9933242797851562f, -12.041450500488281f, -7.16148042678833f, -7.287316799163818f, 5.334017753601074f, -7.423707962036133f, -7.503839015960693f, -13.0750150680542f, -1.5110211372375488f, -3.3898580074310303f, 6.609547138214111f, -1.1499853134155273f, 6.6649370193481445f, -4.917843341827393f, -8.266522407531738f, 16.943111419677734f, 0.35688164830207825f, 2.3017640113830566f, -2.517993927001953f, -20.720897674560547f, -5.1919379234313965f, 6.862222194671631f, 8.344823837280273f, -3.240790367126465f, -0.7951709032058716f, -7.633636951446533f, 1.9410372972488403f, -3.9540672302246094f, -8.897167205810547f, 9.550298690795898f, -12.049423217773438f, 3.2460708618164062f, -0.7465460896492004f, -7.343610763549805f, -25.89638328552246f, 1.305576205253601f, 0.7940993309020996f, 15.786218643188477f, -5.214616298675537f, 5.427506923675537f, -9.456047058105469f, -4.363546371459961f, 14.75342845916748f, -8.062644958496094f, 12.066773414611816f, -17.939416885375977f, -0.6330687403678894f, -8.203537940979004f, -15.027321815490723f, -7.841004848480225f, -6.411040306091309f, -22.517040252685547f, -4.999447822570801f, -8.532695770263672f, 8.279166221618652f, 7.497697830200195f, 8.378273963928223f, -2.62188458442688f, -3.915342330932617f, 3.5234627723693848f, -6.344028949737549f, 7.4199934005737305f, -4.750202655792236f, 1.74264395236969f, 8.607345581054688f, -10.318155288696289f, -0.5510790944099426f, 24.487709045410156f, -4.434352397918701f, 3.7080373764038086f, -16.31922149658203f, 2.130758285522461f, -1.8111369609832764f, 16.641450881958008f, -15.62347412109375f, 9.540854454040527f, -3.6384425163269043f, 1.0087566375732422f, 2.4287056922912598f, 1.3515430688858032f, -1.9539055824279785f, 7.968528747558594f, 3.1463985443115234f, 4.322893142700195f, -4.588891506195068f, 7.193646430969238f, 3.6794357299804688f, -3.6867868900299072f, -12.593381881713867f, 22.773561477661133f, -2.592869997024536f, -8.03166389465332f, 1.6411826610565186f, -5.852877140045166f, 6.320628643035889f, -1.24896240234375f, 2.633263349533081f, 2.3571205139160156f, 2.6310131549835205f, -4.19133996963501f, 9.353876113891602f, -15.862900733947754f, -11.460186958312988f, -0.3524762690067291f, 3.5821545124053955f, -4.435649871826172f, -1.1791728734970093f, 2.729346990585327f, -1.2911462783813477f, -0.1667870581150055f, 3.152031421661377f, -20.307052612304688f, -16.968524932861328f, -2.394564628601074f, -4.503590106964111f, 3.181205987930298f, 0.05041913315653801f, -0.3948255181312561f, -6.3789544105529785f, 1.732195258140564f, -5.6798787117004395f, 7.7667107582092285f, 9.2898530960083f, -4.316888332366943f, -16.857954025268555f, -5.379256725311279f, 4.286224842071533f, 4.1023759841918945f, -8.238680839538574f, 9.628629684448242f, -2.3275868892669678f, 2.8554039001464844f, -8.514399528503418f, -9.410696029663086f, 4.0246405601501465f, -6.667844295501709f, -5.36508321762085f, 0.8656852841377258f, 2.9765312671661377f, -0.6338213682174683f, -4.245971202850342f, 6.7638702392578125f, 8.446202278137207f, -2.508578300476074f, -0.5704231858253479f, 0.46178925037384033f, 10.725545883178711f, 4.313973426818848f, -6.148879528045654f, 3.38698410987854f, -12.07677173614502f, 8.890965461730957f, 9.622424125671387f, -1.9190295934677124f, 1.035591721534729f, 3.041085720062256f, 11.527141571044922f, 2.2250008583068848f, 0.06786001473665237f, 11.618230819702148f, 0.2951813340187073f, 7.107181549072266f, -13.101400375366211f, -4.043383598327637f, 5.551668643951416f, 0.23836034536361694f, -9.364093780517578f, 0.38196349143981934f, 5.709733486175537f, -3.463144540786743f, 10.42895793914795f, -21.435224533081055f, -22.471851348876953f, 2.9046216011047363f, 5.418637275695801f, -2.009808301925659f, -12.568074226379395f, 5.197458744049072f, 12.429717063903809f, -3.8452060222625732f, -8.332062721252441f, -1.3275021314620972f, 8.634135246276855f, -17.408498764038086f, 4.173489570617676f, 3.602097272872925f, 16.919105529785156f, -14.429887771606445f, -7.9446563720703125f, -15.02901554107666f, 5.9782915115356445f, -2.348747491836548f, 2.622194766998291f, 9.501197814941406f, 6.67450475692749f, 5.712249279022217f, -1.8605291843414307f, -0.874760627746582f, 15.388128280639648f, 2.725569486618042f, -4.520296096801758f, -11.915221214294434f, 3.2612195014953613f, -10.38715934753418f, 2.1763875484466553f, -1.2297403812408447f, 6.042415618896484f, -13.808868408203125f, -5.520578861236572f, -1.4444622993469238f, -7.977278232574463f, 8.31882381439209f, -9.937228202819824f, 7.792935848236084f, -8.416111946105957f, 7.689346790313721f, -4.904921054840088f, -0.9467958211898804f, -2.2417097091674805f, 4.855114459991455f, 12.061532020568848f, -4.05377721786499f, 7.5981645584106445f, -5.726215362548828f, -5.700634956359863f, -3.6235997676849365f, -8.09415340423584f, -8.389449119567871f, 3.4986400604248047f, -0.17018863558769226f, -4.8295674324035645f, 3.72519588470459f, 6.054676532745361f, 2.607694149017334f, -5.292807579040527f, 10.3839111328125f, 2.9594953060150146f, 4.974825382232666f, 30.000545501708984f, -6.096185207366943f, 0.5663076043128967f, 4.246780872344971f, -7.833420753479004f, -6.573525905609131f, -4.979223251342773f, 9.320971488952637f, 2.075157642364502f, 0.24154403805732727f, 13.304895401000977f, -9.484021186828613f, 8.853174209594727f, -7.756874084472656f, 0.7791877388954163f, 13.578595161437988f, -4.487305641174316f, -1.5067740678787231f, -20.156299591064453f, 0.3791326582431793f, -4.492989540100098f, 11.668903350830078f, -3.7837438583374023f, -0.9617695808410645f, -15.763601303100586f, 12.063730239868164f, 0.8532925248146057f, -9.071452140808105f, 3.879969358444214f, 14.453145027160645f, -2.9419867992401123f, 8.816530227661133f, 6.016716957092285f, -15.384603500366211f, 7.9487481117248535f, 5.712857723236084f, 6.595734596252441f, -8.348841667175293f, -7.271892070770264f, 8.39649772644043f, 3.010307550430298f, -7.354966640472412f, 4.326541900634766f, 6.269808292388916f, -2.21332049369812f, 2.8673322200775146f, -11.210542678833008f, 8.31998062133789f, -12.9059419631958f, 8.617777824401855f, -3.5901377201080322f, -10.140533447265625f, 0.8802559971809387f, -9.837919235229492f, -9.109603881835938f, -15.81827449798584f, 4.78360652923584f, -1.1366561651229858f, -22.160612106323242f, -12.40213680267334f, -18.063514709472656f, -2.445664882659912f, 15.82851505279541f, -4.466148853302002f, 7.4260993003845215f, 10.562188148498535f, 2.9546096324920654f, 6.616511821746826f, -11.446455001831055f, 1.037653923034668f, 8.754400253295898f, 9.139323234558105f, -6.899957180023193f, -8.12019157409668f, 10.253373146057129f, 3.2580606937408447f, -1.1452258825302124f, 6.390077114105225f, -1.9534237384796143f, -3.5342469215393066f, -6.437942981719971f, -11.188612937927246f, 1.4919053316116333f, 12.896910667419434f, 4.564462184906006f, -8.683686256408691f, 2.0561792850494385f, -7.927750587463379f, -7.966180324554443f, 3.0324785709381104f, -14.164445877075195f, -0.5540791153907776f, 7.670888900756836f, -0.11896628886461258f, 1.1451127529144287f, 22.234376907348633f, -5.0322980880737305f, 10.48377513885498f, 10.193860054016113f, -6.44713020324707f, 4.20560359954834f, -19.378570556640625f, 7.226066589355469f, -11.02578067779541f, 16.957338333129883f, 5.088015079498291f, 7.4178314208984375f, 0.37720149755477905f, 17.275354385375977f, 5.082028865814209f, 0.11316994577646255f, 10.450178146362305f}; - -// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel). -// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad -// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs. -// quad ptr for (m_tile mt, k_group g) = gemm_Apack_dram + (mt*(K/4)+g)*64. -static float gemm_Apack_dram[64*64] __attribute__((section(".data"))) = -{1.9269152879714966f, 1.4451338052749634f, 1.931160569190979f, 1.7743884325027466f, -0.685481607913971f, 0.4334380030632019f, 0.6663737893104553f, -1.3838845491409302f, -1.363769769668579f, -0.8032152652740479f, -1.2786966562271118f, 0.3220294117927551f, 0.34436315298080444f, -0.22056575119495392f, -0.7111086845397949f, -0.447700172662735f, 1.4872840642929077f, 0.8564125299453735f, 1.0118638277053833f, -0.921550989151001f, 0.5635589957237244f, -0.7171905636787415f, -0.07426456362009048f, 0.4868715703487396f, 0.1929578185081482f, -1.1209005117416382f, -0.03842746838927269f, 0.44605663418769836f, -3.1016058921813965f, 0.7118127346038818f, -0.3866667151451111f, -0.7288169264793396f, 0.9007171988487244f, 2.218075752258301f, -1.4364064931869507f, 0.9624499082565308f, -1.507175087928772f, 1.055369257926941f, -0.20960482954978943f, -1.0020296573638916f, -0.6103342771530151f, 0.1956406533718109f, 1.9138009548187256f, 1.5229592323303223f, -1.4587225914001465f, 0.3415871560573578f, 0.9578314423561096f, -0.16066236793994904f, -2.1055209636688232f, 0.5231655240058899f, -1.1298598051071167f, -0.33701515197753906f, -1.610666036605835f, -1.4533969163894653f, 0.16632132232189178f, 0.0329488180577755f, 0.16323445737361908f, -0.7815181016921997f, 0.3378446102142334f, 1.2804899215698242f, -1.4318257570266724f, 1.5885895490646362f, -0.8225308656692505f, -0.32063713669776917f, 0.6784184575080872f, 0.34664666652679443f, -0.1360345333814621f, -1.1753336191177368f, -1.4790465831756592f, 0.46515071392059326f, 1.4703037738800049f, -0.42919591069221497f, 1.51017165184021f, -1.7898789644241333f, 0.12505611777305603f, -0.11616043001413345f, -0.6071268916130066f, -0.3488781750202179f, -2.3908050060272217f, -0.6307737827301025f, -1.2345448732376099f, -0.19733144342899323f, 1.6354095935821533f, 0.35805708169937134f, 0.4322742819786072f, 0.37139150500297546f, -0.9390866756439209f, -0.9817978739738464f, 0.21230429410934448f, -0.26157355308532715f, -0.7621514797210693f, 1.3705363273620605f, -0.25973787903785706f, -0.45791950821876526f, 0.322247713804245f, -0.788766622543335f, -0.04306747764348984f, -1.0545889139175415f, 0.6547407507896423f, 0.47876790165901184f, -0.1250254064798355f, -0.004656785633414984f, -0.6013189554214478f, -0.6420586109161377f, -0.7252011299133301f, -0.44025033712387085f, -1.190559983253479f, -0.4809381365776062f, -0.7190185785293579f, -1.2322070598602295f, 1.875388741493225f, 1.3061575889587402f, -1.6046669483184814f, 1.2779955863952637f, 0.5760045647621155f, 1.353700041770935f, 0.7821183800697327f, 0.07954943925142288f, -0.09964022785425186f, 0.8265887498855591f, -0.9527732729911804f, 2.1848294734954834f, 0.7756073474884033f, -0.9903622269630432f, -0.38583096861839294f, -0.598077118396759f, 1.1042989492416382f, -0.9275763630867004f, -0.7521352767944336f, -0.1721901297569275f, 1.1415079832077026f, 0.5260620713233948f, -1.598767638206482f, 0.3781784772872925f, -0.9851518273353577f, 1.591395616531372f, 0.5216943025588989f, -0.48009708523750305f, 0.455719918012619f, -1.3641812801361084f, 0.5233525037765503f, -0.28154700994491577f, -0.5223758816719055f, -0.26273947954177856f, 1.6487230062484741f, 0.5237884521484375f, 0.018564576283097267f, 2.1120378971099854f, -0.10912995040416718f, 0.7051141262054443f, -2.488459348678589f, -0.1208132952451706f, -0.46386733651161194f, -1.2871733903884888f, 0.2503303289413452f, 0.008205652236938477f, -0.8211768269538879f, 0.05281926319003105f, -0.7401803731918335f, 0.9314952492713928f, -0.3924786448478699f, 0.056621819734573364f, -1.8058050870895386f, -0.5207571387290955f, 0.7151994705200195f, -1.7236974239349365f, -0.33131900429725647f, -0.48302069306373596f, 0.18237744271755219f, 0.7388824224472046f, -1.3610970973968506f, -0.40586018562316895f, -0.47086891531944275f, 0.42497751116752625f, 0.16235657036304474f, -0.4593467116355896f, -1.4036071300506592f, 0.4262961447238922f, 0.9254348874092102f, -0.9320061206817627f, 0.03913922235369682f, -0.8434810638427734f, 0.8435799479484558f, 0.11329790204763412f, -0.38666075468063354f, 0.03389474004507065f, 1.8018341064453125f, -0.7110859751701355f, 0.6016423106193542f, 0.4825834333896637f, -0.2369976043701172f, -0.9419456720352173f, -0.7278813123703003f, 0.575005054473877f, -0.3753443658351898f, 0.18516133725643158f, 1.305860161781311f, 0.4351435601711273f, 0.9874473810195923f, 0.0771508663892746f, -1.7906768321990967f, -0.31229403614997864f, -0.07434194535017014f, -0.3495793640613556f, -0.28251126408576965f, 0.48813387751579285f, 0.5099347233772278f, -0.7089186310768127f, -0.5594301819801331f, -0.6417241096496582f, 1.0330873727798462f, 1.0686918497085571f, 0.24659274518489838f, 0.26588720083236694f, -0.33197471499443054f, -0.9228128790855408f, 0.09329313784837723f, -0.2541753351688385f, -0.15664155781269073f, 0.3797488212585449f, 0.7692679762840271f, 1.0082393884658813f, 1.670624852180481f, 2.1860759258270264f, -0.7688388824462891f, -2.2063984870910645f, -0.6866509318351746f, 1.3065344095230103f, -1.9775909185409546f, -0.5870985388755798f, -0.8076189756393433f, -1.2619991302490234f, -1.9152568578720093f, -1.205536127090454f, -0.8708454966545105f, 0.9993040561676025f, -0.7668924331665039f, -0.595004141330719f, 1.5921050310134888f, -0.6493158936500549f, 0.7624453902244568f, -0.7508030533790588f, 0.6368136405944824f, 0.4598345160484314f, 0.01789604313671589f, 0.0826888456940651f, 0.824364185333252f, 1.0860532522201538f, -0.6421752572059631f, -0.9542103409767151f, -0.6410972476005554f, 1.2751853466033936f, -0.9494866728782654f, 0.3926331400871277f, -0.41619211435317993f, 0.45214036107063293f, 1.6423169374465942f, 0.01086814422160387f, -0.9726738929748535f, -0.8146268725395203f, -1.3793021440505981f, 0.8853808045387268f, 0.024699924513697624f, 1.096641182899475f, 1.3438509702682495f, 0.061276569962501526f, -0.414562851190567f, 0.9594927430152893f, 0.01691739819943905f, 0.8229668736457825f, 1.861944556236267f, 0.8520749807357788f, -0.1595974713563919f, -0.33874234557151794f, 0.9584577679634094f, -1.0212392807006836f, 0.625802755355835f, 0.1824439913034439f, -1.0641486644744873f, -0.6836934685707092f, -1.2922308444976807f, 0.08526104688644409f, -0.6902380585670471f, 0.10350999981164932f, 0.08027740567922592f, -0.886031985282898f, -1.077892780303955f, -1.6946725845336914f, -0.4973975419998169f, -1.3406795263290405f, 1.6192004680633545f, -0.49492356181144714f, -2.5849502086639404f, 0.7863810062408447f, -0.7601934671401978f, 0.06604336202144623f, 0.766244113445282f, 0.7481252551078796f, -0.22995619475841522f, 0.8290349841117859f, 0.7448412775993347f, 1.4801039695739746f, 0.8848565220832825f, 1.1805996894836426f, 0.439589262008667f, -0.5853705406188965f, 1.450609803199768f, -0.5922516584396362f, -0.02399955503642559f, -0.057920295745134354f, -0.4075061082839966f, -0.0007737990817986429f, 0.64540034532547f, -0.16356196999549866f, -2.172283887863159f, 2.0921294689178467f, 1.345484972000122f, 0.8391514420509338f, -0.8342104554176331f, -2.8929238319396973f, -0.7581311464309692f, 0.5361881256103516f, 0.2694815397262573f, 0.15431594848632812f, -0.1221928745508194f, 0.5666652917861938f, 0.9623646140098572f, 0.1620604395866394f, 0.353316068649292f, -0.9085567593574524f, 0.08768323808908463f, 0.7953095436096191f, 0.12682189047336578f, -0.20004984736442566f, 1.0300744771957397f, -0.3875778615474701f, 1.078317642211914f, 0.5246226191520691f, -0.21037597954273224f, 0.4407670795917511f, -0.7469954490661621f, -0.7097623348236084f, -0.14264194667339325f, 1.195958137512207f, -2.6474881172180176f, 0.3129958212375641f, 1.0937845706939697f, 0.2792847752571106f, -2.4520716667175293f, 0.9949536919593811f, -0.8680985569953918f, -0.7124031782150269f, 0.8008005619049072f, 1.1412016153335571f, -0.7328027486801147f, -0.14829230308532715f, 1.7093087434768677f, -0.4875054359436035f, 0.15271379053592682f, -1.3061535358428955f, -1.4575366973876953f, 0.8050477504730225f, -0.1177205815911293f, 0.1864478439092636f, 0.4159761369228363f, 0.7201864719390869f, -0.5701602697372437f, -1.6171332597732544f, 1.680620551109314f, 0.05164359509944916f, 0.10429783165454865f, -2.3184432983398438f, 0.05792269483208656f, 0.050095997750759125f, -0.0388023778796196f, -1.4039719104766846f, -0.9712379574775696f, -1.1133604049682617f, -0.29864323139190674f, 0.3547132909297943f, 1.9025356769561768f, -0.13413065671920776f, 0.32332202792167664f, -0.35899198055267334f, 1.27912437915802f, 0.7439519762992859f, 0.3487516939640045f, -0.39799532294273376f, 1.1929805278778076f, 0.6084084510803223f, 0.9446058869361877f, -1.0597201585769653f, 0.25403109192848206f, 0.4981626570224762f, -0.9536206126213074f, 0.09063850343227386f, -0.7346699833869934f, -1.4067999124526978f, 1.1284750699996948f, 0.051366694271564484f, 1.2964228391647339f, -0.4815843999385834f, 0.9675941467285156f, 1.0804862976074219f, 1.9372931718826294f, 1.6308681964874268f, -1.5824053287506104f, 0.3057299852371216f, -0.1790592074394226f, -1.1999552249908447f, -0.09247277677059174f, 1.7422553300857544f, 0.044657133519649506f, -2.3609628677368164f, -1.2123126983642578f, 0.6950237154960632f, 0.610466480255127f, -1.0494661331176758f, -0.46568843722343445f, -1.7808643579483032f, 0.7287133932113647f, -0.08472306281328201f, 0.9871290922164917f, 0.4150581359863281f, 1.1992844343185425f, 0.12711313366889954f, -1.01665461063385f, -1.2660012245178223f, -1.5211198329925537f, -0.2904934287071228f, 2.602391004562378f, 1.835181474685669f, 1.334737777709961f, 0.603898823261261f, 1.6047972440719604f, 1.5080454349517822f, 0.9808937907218933f, 1.0844124555587769f, 1.1456739902496338f, -0.7174144983291626f, -0.4292171895503998f, 0.4403660297393799f, -0.007675689645111561f, 0.38916081190109253f, 0.3478375971317291f, -0.13345853984355927f, -0.09572362899780273f, -1.9180361032485962f, -0.2316243201494217f, -1.7222950458526611f, -2.4801201820373535f, 0.30942854285240173f, 0.41459226608276367f, 0.9477656483650208f, -0.14181147515773773f, 2.833967924118042f, 1.010284185409546f, 0.6377718448638916f, -0.518220841884613f, 0.34287506341934204f, 0.7401772737503052f, -0.15693345665931702f, -0.08114803582429886f, -1.3923954963684082f, 0.041759490966796875f, -0.827768862247467f, -0.4175437390804291f, -0.5003090500831604f, 1.15656316280365f, -0.676629364490509f, -0.2763414680957794f, 1.9534740447998047f, 0.6110401153564453f, 0.15978877246379852f, 0.83954256772995f, -1.4590637683868408f, 1.4161995649337769f, 1.138344645500183f, 1.2586976289749146f, 0.540465772151947f, -0.2515752911567688f, 1.334702968597412f, -1.1954537630081177f, 1.0350031852722168f, 0.2690545618534088f, -0.5730168223381042f, -0.19321373105049133f, 2.0486814975738525f, 1.2208385467529297f, 1.7697970867156982f, 0.05852266773581505f, -1.4936561584472656f, 0.6833979487419128f, -0.2505214214324951f, 0.8691263794898987f, 0.4350730776786804f, 0.859858512878418f, 0.48353928327560425f, 0.8123369216918945f, 1.6896470785140991f, -0.036629438400268555f, -0.3303174376487732f, 0.7767809629440308f, -1.0880382061004639f, -0.6076440215110779f, 0.6268176436424255f, -1.6682480573654175f, -0.22138521075248718f, -0.13825182616710663f, 1.6704555749893188f, -0.9609367251396179f, -2.2717032432556152f, -1.3846737146377563f, -2.5095443725585938f, -1.9005532264709473f, -0.004505051765590906f, 0.9732939004898071f, -0.7939430475234985f, 0.6838752627372742f, 1.621694564819336f, -1.7376028299331665f, -1.8736529350280762f, 2.129624843597412f, 0.22523505985736847f, 0.9212995171546936f, -0.545271098613739f, 0.05182264745235443f, -0.13386189937591553f, -0.8712361454963684f, 0.4880010485649109f, 0.22857652604579926f, 1.666792392730713f, -1.0150787830352783f, 0.3752319812774658f, -1.3245893716812134f, 0.8512656688690186f, -0.12535162270069122f, 2.3259060382843018f, -1.5181471109390259f, -0.07724537700414658f, 0.5282443761825562f, -2.15816330909729f, -0.3284812867641449f, -0.058557309210300446f, -0.223365917801857f, 0.7845868468284607f, 0.02485940419137478f, 0.15392017364501953f, -0.5419175624847412f, 0.08790969103574753f, -0.5160817503929138f, -0.40046969056129456f, -1.3658148050308228f, -0.9203909635543823f, 0.1387282907962799f, 0.9856945276260376f, -0.008228386752307415f, -1.6607975959777832f, -2.247206211090088f, 0.12574470043182373f, 1.7173614501953125f, 0.02864718623459339f, -0.34595024585723877f, -1.0602530241012573f, -0.44102486968040466f, -1.241483449935913f, 0.6001842617988586f, -0.6088271737098694f, 1.111746072769165f, 0.6661149263381958f, -1.1797568798065186f, 1.2783364057540894f, -1.4493319988250732f, -0.6637441515922546f, -0.4478967487812042f, -0.5525766611099243f, 0.3188803195953369f, 0.640755295753479f, 0.2868328094482422f, -0.572657585144043f, -0.3136177957057953f, -0.32025346159935f, -0.4702208340167999f, -0.508095383644104f, -0.6227966547012329f, -0.44026491045951843f, -0.5297411680221558f, 0.28815189003944397f, -0.605182409286499f, 0.3657907545566559f, 0.4234687089920044f, 0.07448001205921173f, -0.42451897263526917f, 0.5832474231719971f, -0.7308424115180969f, 0.0835680365562439f, -0.12925422191619873f, -0.844377875328064f, -0.6086435317993164f, -0.6184902191162109f, -0.7891808748245239f, -2.3179564476013184f, 0.9625157713890076f, 0.869049608707428f, -0.17924511432647705f, -0.39920157194137573f, -0.3874586224555969f, -0.1492866724729538f, 0.3057209253311157f, 1.0669267177581787f, 0.17482025921344757f, 0.39990535378456116f, -0.7149624228477478f, -0.5513465404510498f, -0.046192023903131485f, -1.647040605545044f, -0.167823925614357f, 1.294582724571228f, 0.2794382870197296f, -0.8097057938575745f, 0.19955870509147644f, 0.49674081802368164f, -0.22963792085647583f, -0.5522539615631104f, -0.7745925188064575f, -0.4501533806324005f, -1.0939292907714844f, 1.989207148551941f, -0.047562163323163986f, 1.9889612197875977f, -1.6457397937774658f, -1.0362098217010498f, 1.6433145999908447f, 0.22267311811447144f, -0.5718191266059875f, -1.4298604726791382f, -1.2461947202682495f, -2.369169235229492f, -0.40709349513053894f, -0.09342008084058762f, -1.5575724840164185f, -0.18526747822761536f, -1.6021603345870972f, -0.07198750972747803f, 2.0207436084747314f, 1.900311827659607f, -0.4833274185657501f, -0.4503057301044464f, 2.0070879459381104f, -0.8483412265777588f, -2.7936289310455322f, 0.45901596546173096f, -0.41459938883781433f, -0.5614708065986633f, 0.8702965974807739f, -1.0284309387207031f, 0.9956361055374146f, 0.7527588605880737f, 1.3528969287872314f, -0.906094491481781f, 0.25391900539398193f, 1.6950805187225342f, -0.740294337272644f, -0.0729660615324974f, -1.2531019449234009f, 1.6489421129226685f, -0.7111545205116272f, 0.5309328436851501f, 1.4558700323104858f, -0.5949130654335022f, -1.0552809238433838f, 0.40444278717041016f, -0.8797858357429504f, 0.4047577977180481f, 1.288827657699585f, -2.0487122535705566f, 0.9364385008811951f, 0.028089528903365135f, 0.31428107619285583f, -0.5479549169540405f, 1.118869423866272f, 1.6005686521530151f, 0.5235219597816467f, -1.3614802360534668f, 0.3316534161567688f, 1.2687278985977173f, -1.3284013271331787f, 2.1425962448120117f, -0.6011420488357544f, 0.17846599221229553f, 0.05229547247290611f, -1.0810555219650269f, 0.7122363448143005f, -0.17536965012550354f, 0.1415553092956543f, -1.1425532102584839f, 1.7732776403427124f, -0.07858924567699432f, -1.71055006980896f, 1.9562491178512573f, -1.00010085105896f, 1.2904434204101562f, 0.7060741186141968f, -0.5153723955154419f, -1.2741512060165405f, 0.2649095058441162f, -1.5468504428863525f, 0.01762307994067669f, -0.031765542924404144f, -1.7734959125518799f, 1.0348176956176758f, -0.44875210523605347f, -2.071660280227661f, 0.43104586005210876f, 0.8384853601455688f, 1.7684898376464844f, -0.6919524669647217f, -1.1755682229995728f, 0.35730111598968506f, 1.0827196836471558f, 2.1227850914001465f, 1.2731683254241943f, 0.7567060589790344f, 0.0782259851694107f, 0.10164086520671844f, -0.7046414017677307f, -0.626437783241272f, -0.03045438416302204f, -0.4125255346298218f, 0.3683530390262604f, -0.2698453664779663f, -0.9857985377311707f, -0.47199076414108276f, -0.0783226415514946f, 0.5892837643623352f, 1.2498642206192017f, -1.234653115272522f, -0.0013108636485412717f, 0.7755194902420044f, 0.19315829873085022f, 1.3433040380477905f, -0.39465200901031494f, -0.5150922536849976f, 0.3830311596393585f, -0.9769555926322937f, 0.7637977004051208f, 0.12306158244609833f, -1.2370758056640625f, -1.2894343137741089f, -0.9705761075019836f, 0.9187757968902588f, 0.9821351766586304f, -0.4879138767719269f, -0.30360376834869385f, 2.0265355110168457f, 0.40967342257499695f, 0.7132695913314819f, 1.8868111371994019f, 0.6902899742126465f, -0.04476971551775932f, -0.03363388776779175f, 1.1792222261428833f, 0.8757511377334595f, -2.301875114440918f, 1.0762810707092285f, 1.4723693132400513f, 0.6662830114364624f, 0.22690092027187347f, -0.9138230085372925f, -1.457029104232788f, 0.03581761196255684f, -0.9291303157806396f, 0.4038029611110687f, -0.21844321489334106f, -0.4939993619918823f, 1.179942011833191f, 1.8594977855682373f, -0.4137862026691437f, 0.15132997930049896f, -0.0010087001137435436f, -1.0667427778244019f, 1.4108561277389526f, 0.24650610983371735f, 0.4927920699119568f, -0.6581372618675232f, -0.10233523696660995f, 0.12058872729539871f, 0.2761908769607544f, -0.7139783501625061f, 0.16629981994628906f, 1.1366126537322998f, -0.33142781257629395f, 2.6221468448638916f, 0.5184088349342346f, 0.739393413066864f, -0.8494256734848022f, -1.9893426895141602f, -1.3143675327301025f, 0.1328691989183426f, -0.5128253102302551f, 0.07802387326955795f, -0.5991530418395996f, -0.8056637048721313f, -0.5388752818107605f, 0.8337291479110718f, 2.1441681385040283f, -0.46184006333351135f, 0.6495042443275452f, 0.36905255913734436f, -0.7015367746353149f, 0.27310314774513245f, -1.6593921184539795f, 0.29731303453445435f, -1.31621515750885f, 0.12191462516784668f, 0.3006223440170288f, 0.5258087515830994f, 0.4770564138889313f, -0.20757682621479034f, 0.4625823199748993f, -0.9585452079772949f, 1.7045671939849854f, 1.419979453086853f, 0.09495851397514343f, 0.3802972435951233f, -0.4323408901691437f, 2.7312309741973877f, 0.3062905967235565f, 0.4344584047794342f, -1.2524477243423462f, 0.47808775305747986f, 0.07734657824039459f, -0.48799172043800354f, 0.7261772155761719f, -0.9319478273391724f, -0.8718891143798828f, 0.45363426208496094f, 0.3459012508392334f, 0.848518967628479f, -0.7525874376296997f, 0.19898031651973724f, 0.1414770483970642f, 0.43200522661209106f, 1.182044506072998f, 0.0033933203667402267f, -1.5844100713729858f, 0.2761341631412506f, 0.6477669477462769f, 1.1913690567016602f, 0.09115186333656311f, -1.5909662246704102f, -0.027118360623717308f, 1.2460919618606567f, 0.6424751281738281f, -0.047891248017549515f, -0.647229790687561f, -0.23609064519405365f, 0.07110362499952316f, -0.30918216705322266f, 0.32602694630622864f, -1.0240145921707153f, -2.5446670055389404f, -0.5895728468894958f, -0.4324244260787964f, -0.8140076398849487f, -0.3890652060508728f, -1.13597571849823f, -0.3532457649707794f, -2.3065085411071777f, -0.20395424962043762f, 0.6685602068901062f, -1.2822614908218384f, 0.30340856313705444f, 0.5633530616760254f, -0.09658124297857285f, -0.3894469738006592f, 0.22404761612415314f, 1.3719074726104736f, 0.569182813167572f, 1.1740480661392212f, -0.7359927892684937f, 0.5279164910316467f, -0.52259761095047f, 1.4638569355010986f, -1.2868918180465698f, 0.6853673458099365f, 1.0429801940917969f, 1.96529221534729f, -0.45007675886154175f, -0.5786357522010803f, 1.541925072669983f, 2.8543806076049805f, -0.7554785013198853f, -0.5379461050033569f, -0.7911050319671631f, 0.7011352777481079f, -1.4032478332519531f, -0.012685478664934635f, -0.5187733173370361f, 1.255434274673462f, 0.17988650500774384f, -0.13968797028064728f, 0.6899017095565796f, -0.9638485312461853f, 0.47390419244766235f, -1.083811640739441f, -0.108744777739048f, 0.8243650794029236f, 1.3675810098648071f, 0.7378400564193726f, -0.19896702468395233f, 0.6674330234527588f, 0.03600366786122322f, 0.24083632230758667f, -1.5012763738632202f, -0.7149558067321777f, -2.126762628555298f, -1.1807502508163452f, -1.3129348754882812f, -2.5667941570281982f, 0.6503364443778992f, -0.3889259994029999f, -0.4189043343067169f, 0.7983470559120178f, -0.3197358250617981f, -0.8505349159240723f, -1.3615714311599731f, -0.8035953640937805f, -0.06347727030515671f, 0.13253536820411682f, -1.9266542196273804f, 0.8539193272590637f, -0.13408313691616058f, -1.282929539680481f, 0.03780364990234375f, 0.7096128463745117f, 1.1662380695343018f, 0.8126105666160583f, 1.4384385347366333f, 1.8890222311019897f, -0.9130924344062805f, 0.03610055148601532f, -0.5193602442741394f, -1.3776048421859741f, 0.6756148934364319f, 0.7642406225204468f, 0.1278512328863144f, 0.512991189956665f, -1.0407685041427612f, 0.448485791683197f, -1.1702114343643188f, 0.8198426961898804f, 0.01693599671125412f, 1.4981187582015991f, -0.7068426609039307f, 0.5934628248214722f, 1.919209361076355f, 1.3406710624694824f, 0.07648162543773651f, -0.4410470724105835f, -0.0978068932890892f, 1.095009684562683f, 1.0229133367538452f, 0.5397310256958008f, -0.7647228837013245f, -0.590737521648407f, -0.10318559408187866f, 0.6214459538459778f, 0.5325868129730225f, 0.043896086513996124f, -1.2519514560699463f, 0.0696544423699379f, -1.6514869928359985f, 0.9199973940849304f, 0.34005025029182434f, 0.1417587399482727f, 1.8445940017700195f, 0.3398909568786621f, -0.5557951331138611f, 0.5655050277709961f, -0.05528254434466362f, 0.8540631532669067f, 1.1894739866256714f, 0.42318588495254517f, -0.6035352349281311f, 1.4443233013153076f, 3.0250484943389893f, -1.6034338474273682f, 2.1477253437042236f, -0.3787555396556854f, 1.4557304382324219f, 1.1084681749343872f, -1.184537410736084f, 0.7199674844741821f, 0.7042727470397949f, 0.5057917237281799f, 1.204850673675537f, -0.4900680184364319f, 0.7606944441795349f, -0.33889833092689514f, -0.1742597371339798f, 0.23202891647815704f, 1.3462589979171753f, -0.42982181906700134f, -0.6604134440422058f, -1.5597758293151855f, -0.3461014926433563f, 0.5544233322143555f, 1.3835493326187134f, 0.41140761971473694f, 0.7098760008811951f, 0.22245365381240845f, -0.982473611831665f, -0.35945725440979004f, -0.7463049292564392f, 0.5179733633995056f, 0.6092063784599304f, 0.5064983367919922f, 0.8556069731712341f, 0.5761587619781494f, 0.11352583765983582f, -0.8009540438652039f, -0.2633814215660095f, 1.5817502737045288f, -1.2247875928878784f, 0.23484112322330475f, -1.407819390296936f, -0.09603477269411087f, -0.294853538274765f, -0.5165784358978271f, 0.2704370319843292f, -1.0562596321105957f, -0.8496796488761902f, -0.2919580638408661f, 0.10698884725570679f, -0.9970720410346985f, 0.05950453504920006f, 0.6107109189033508f, 0.5484238862991333f, 0.7350953221321106f, 0.962885856628418f, 0.08861476927995682f, -0.0801108106970787f, 1.0644340515136719f, -0.2798626720905304f, -0.6881742477416992f, -1.1818499565124512f, 0.24130821228027344f, -0.6987038254737854f, 0.45643651485443115f, -0.10149910300970078f, -0.34807199239730835f, 1.7285562753677368f, 0.2142704576253891f, 0.45765289664268494f, -0.269319087266922f, -1.5785412788391113f, -0.3476867079734802f, 0.5194124579429626f, 1.4888246059417725f, 1.0837124586105347f, 0.4755038022994995f, -1.0340323448181152f, 0.18275369703769684f, -0.20516234636306763f, -0.3147018849849701f, 0.08301469683647156f, 0.2176828533411026f, 0.2920781075954437f, -0.17470814287662506f, 0.9676891565322876f, 0.45085909962654114f, 0.6715953946113586f, 0.8490674495697021f, 1.170888900756836f, 0.8825610876083374f, 0.17298388481140137f, -1.4316335916519165f, -0.4947497546672821f, 0.6246524453163147f, -0.7811664342880249f, -0.41328999400138855f, 0.712009072303772f, 1.1277714967727661f, -0.6925867199897766f, -1.7561272382736206f, -0.6673583388328552f, 0.04091703146696091f, -0.060151856392621994f, 0.2014705091714859f, 2.1779797077178955f, -0.23840203881263733f, 0.5123522281646729f, 0.1427735835313797f, -0.8951197266578674f, -0.7939775586128235f, 0.6872723698616028f, 0.3946205973625183f, -0.9005926847457886f, -1.5005147457122803f, -0.8442767858505249f, 1.4259073734283447f, 1.6183183193206787f, -0.06933283060789108f, 0.06978437304496765f, 0.3839779198169708f, 1.7791979312896729f, 0.5468734502792358f, -0.9818529486656189f, 0.6328914165496826f, 0.5340191125869751f, -0.674835205078125f, 0.7835897207260132f, 1.1304746866226196f, 0.8906894326210022f, -0.2404831200838089f, -0.32920387387275696f, 0.5127183794975281f, -0.26440465450286865f, 0.2585761547088623f, -1.6634936332702637f, 1.2309634685516357f, 0.25832492113113403f, -0.06058019772171974f, 1.1258721351623535f, -1.048923373222351f, -0.88661128282547f, -0.3876877427101135f, -1.1108732223510742f, 0.8258382678031921f, 0.47655177116394043f, -0.4854878783226013f, -0.1140289157629013f, -0.4026731848716736f, 1.354137897491455f, 0.4775455594062805f, -0.7650561332702637f, 1.2286686897277832f, -2.4340736865997314f, -0.5304896831512451f, 0.25538599491119385f, -0.5224623084068298f, 0.7963698506355286f, 0.44965043663978577f, -3.106328010559082f, 0.9458276629447937f, -0.8396266102790833f, -0.06616523861885071f, -0.8452204465866089f, 1.9770677089691162f, -0.07092823088169098f, -1.4889180660247803f, 1.2306435108184814f, 0.7042104005813599f, -0.3497500419616699f, -2.0363707542419434f, -0.4588965177536011f, -1.1338030099868774f, 0.6588250994682312f, 0.3726101517677307f, -0.9897713661193848f, -0.15447334945201874f, 0.33320167660713196f, -0.9029294848442078f, 0.3004419207572937f, 0.0267170500010252f, -0.3697128891944885f, -1.2930028438568115f, 0.4252126216888428f, -0.05628490820527077f, -1.338055968284607f, 0.5246880650520325f, -0.9283785820007324f, -0.14128278195858002f, -0.2966694235801697f, -1.9104946851730347f, -0.6022037863731384f, -1.6013476848602295f, -1.2525506019592285f, 0.6440190076828003f, 1.6395184993743896f, -0.25020739436149597f, -0.24597389996051788f, 0.8795658946037292f, -0.016383398324251175f, -1.4897207021713257f, -0.4389103353023529f, -0.6970252990722656f, -0.17175325751304626f, -0.6456266641616821f, 0.23320063948631287f, 0.26085028052330017f, -0.7153372168540955f, -0.059470854699611664f, -0.5745509266853333f, 0.7591805458068848f, -1.0744175910949707f, -0.09136287868022919f, 0.3293827772140503f, 0.07306429743766785f, -0.10749480873346329f, -1.5194628238677979f, -0.5850174427032471f, -0.08793152123689651f, -0.6866653561592102f, 0.4101375937461853f, -1.3473118543624878f, 1.4177610874176025f, -0.467404842376709f, -0.9928666353225708f, -1.9059150218963623f, -2.0202691555023193f, 0.32121846079826355f, -0.5283262729644775f, 0.6973733305931091f, 0.9967362284660339f, -1.3085604906082153f, 0.32580918073654175f, 1.8071491718292236f, -0.27431318163871765f, -0.1326882690191269f, 0.32671934366226196f, -0.8231000304222107f, 0.6738032102584839f, 0.551419734954834f, 1.1634254455566406f, -0.9665390253067017f, -0.6739510893821716f, 0.2892301678657532f, -0.46433013677597046f, 0.8342204093933105f, 1.0237910747528076f, 0.659813642501831f, -1.458429217338562f, -0.7326241135597229f, 1.2922906875610352f, 1.6295740604400635f, -0.8344282507896423f, -0.539470374584198f, 1.4665507078170776f, 2.654942035675049f, 1.609459400177002f, 0.367727130651474f, -0.9192121624946594f, 0.35196706652641296f, -0.15669254958629608f, -1.098692774772644f, -1.2132517099380493f, -0.07032525539398193f, 1.8989076614379883f, 0.4093967378139496f, -1.4458993673324585f, -1.5456795692443848f, -0.4921732246875763f, 0.09215065091848373f, -1.1077474355697632f, 1.0582451820373535f, -0.29416555166244507f, -0.5785751342773438f, 1.212047815322876f, 2.0987792015075684f, -1.5964131355285645f, 0.5674765110015869f, 0.9744538068771362f, 0.27448296546936035f, -0.04056643322110176f, -0.5840954780578613f, -0.31466683745384216f, -0.16959930956363678f, 0.658042311668396f, 0.8539144992828369f, -0.7443782091140747f, -0.1468161642551422f, 1.0819309949874878f, 1.2373405694961548f, -1.3463493585586548f, -0.5285679697990417f, -1.5322294235229492f, 0.8283826112747192f, 0.45322731137275696f, -0.3450125455856323f, -0.2933650016784668f, 0.10613418370485306f, 0.11260014772415161f, 0.02781728096306324f, 0.5361921191215515f, -1.0331605672836304f, 1.0188977718353271f, -0.8913255929946899f, 0.8866236805915833f, 0.8713390231132507f, -0.4831593930721283f, -1.7715193033218384f, 0.8562206029891968f, -3.832531690597534f, -0.9927570819854736f, -0.11961783468723297f, 1.397810459136963f, -0.3067088723182678f, -1.4679176807403564f, 0.0910743772983551f, 1.2350000143051147f, 0.5837976336479187f, -1.8317182064056396f, 0.1937909871339798f, -0.8611426949501038f, -0.522757887840271f, 1.7186238765716553f, 0.0968942642211914f, 0.4322175085544586f, 0.11584769189357758f, -0.2557562291622162f, 1.1861584186553955f, -0.9166569709777832f, 0.8642276525497437f, -1.716816782951355f, 0.6718529462814331f, -0.21214154362678528f, 1.0166347026824951f, 0.2806217670440674f, 1.9681813716888428f, -0.27264782786369324f, 1.2400078773498535f, -0.5684311389923096f, 0.36735019087791443f, 0.24113479256629944f, 1.9915446043014526f, -0.9585699439048767f, -1.2203160524368286f, -0.7793720364570618f, -1.0658658742904663f, -0.5502451658248901f, 0.9851812124252319f, 1.387345790863037f, -0.794191300868988f, 0.6909231543540955f, -0.7403607368469238f, 0.9804211258888245f, -0.905766487121582f, -2.9151322841644287f, 0.47318676114082336f, -0.05469474941492081f, 1.0238094329833984f, -0.5702749490737915f, 0.2909986078739166f, -0.41753849387168884f, -1.0129939317703247f, 0.535078763961792f, -0.7609738707542419f, -0.8248465657234192f, 0.36742645502090454f, 0.7183824777603149f, -0.8665743470191956f, -0.17533333599567413f, 0.768028736114502f, 1.0834310054779053f, 0.5876799821853638f, 0.04771281033754349f, 2.1327197551727295f, 0.19281719624996185f, -0.07964225858449936f, 1.1059718132019043f, -0.9939178228378296f, -1.3392163515090942f, -1.2726119756698608f, 0.3544987440109253f, 0.42930465936660767f, -0.5719326138496399f, -0.30639445781707764f, -0.12276917695999146f, 1.62221360206604f, 0.07731132954359055f, 0.183979332447052f, -0.8637551665306091f, 0.23347528278827667f, 0.3847457766532898f, 1.3200364112854004f, 0.2528532147407532f, 2.9082677364349365f, 1.2357676029205322f, -0.6267421841621399f, -0.28073710203170776f, 2.0288591384887695f, -0.46663370728492737f, -0.5359372496604919f, 0.7414069175720215f, 0.08158037811517715f, 1.2316521406173706f, -0.8425887823104858f, -1.141897201538086f, 1.2004878520965576f, -0.8595814108848572f, -1.5196866989135742f, -0.10754015296697617f, 1.4483332633972168f, -2.0370566844940186f, 1.3712586164474487f, -0.243259459733963f, -0.7683921456336975f, 0.1017654687166214f, -0.35750812292099f, 0.35395190119743347f, 0.20281589031219482f, 2.9194436073303223f, -1.6971954107284546f, -0.8292406797409058f, -1.2668757438659668f, 1.2814747095108032f, -0.29335519671440125f, 0.7705280780792236f, -0.5614521503448486f, 1.4171453714370728f, 0.23598231375217438f, -0.29366371035575867f, -1.2262547016143799f, 0.38033702969551086f, -1.2398927211761475f, -0.5345798134803772f, 0.33024173974990845f, 1.9377914667129517f, 1.0871516466140747f, 1.3149042129516602f, -0.9444983005523682f, 0.1276102066040039f, 2.106604814529419f, -1.1304327249526978f, -0.9464563131332397f, 0.1686755269765854f, -0.446566104888916f, -0.6286743879318237f, -0.11334973573684692f, -1.962886095046997f, -1.523536205291748f, 0.645367443561554f, -0.9533721804618835f, -0.5536230206489563f, 0.6801379919052124f, 1.2470786571502686f, -0.1793764978647232f, 0.6101059317588806f, -0.10875027626752853f, 0.996456503868103f, -0.7419731020927429f, -1.1421011686325073f, -1.1778020858764648f, -0.04226749762892723f, 0.30849045515060425f, -0.7805798053741455f, -0.8158796429634094f, -2.9890658855438232f, 1.5734566450119019f, -1.3029974699020386f, 1.1616994142532349f, -0.25818052887916565f, 0.31204938888549805f, -1.2942535877227783f, 0.608341634273529f, -1.1809622049331665f, 0.15562251210212708f, 0.6069639921188354f, 1.4125137329101562f, -0.27004849910736084f, -0.4410642385482788f, -0.1343953162431717f, 1.1372771263122559f, 0.18370741605758667f, 1.8697383403778076f, 1.0696011781692505f, -0.17657233774662018f, -1.340453863143921f, 0.717379093170166f, 0.6038413643836975f, 0.7894347310066223f, 0.9626035690307617f, -0.25843867659568787f, -0.8331825137138367f, -0.02316661737859249f, 1.4387904405593872f, -0.77969890832901f, -0.3609391152858734f, 0.21925963461399078f, -0.4726978540420532f, -1.0638948678970337f, -0.4561823606491089f, 0.5214431285858154f, -0.5561836957931519f, -0.2302778661251068f, 1.117246150970459f, 0.7824702858924866f, -1.1049346923828125f, -0.7501540780067444f, -0.47921040654182434f, -0.011093219742178917f, 0.03258634731173515f, -0.756567120552063f, 0.104627326130867f, 0.4133651852607727f, -0.958824634552002f, -0.2272576242685318f, 1.3634916543960571f, -2.3571479320526123f, 1.173104166984558f, 0.7147674560546875f, 0.4163925051689148f, -0.06465863436460495f, -0.7909473180770874f, 1.2354754209518433f, 0.2998451292514801f, -0.9952824711799622f, -0.5479734539985657f, 1.514818549156189f, -0.325872004032135f, 0.0061524491757154465f, -1.5123530626296997f, 0.25006136298179626f, -2.4219810962677f, -0.8351162672042847f, 0.6859942674636841f, 1.443666696548462f, 0.6747081875801086f, -0.0002302070497535169f, -0.21609316766262054f, 1.0141247510910034f, 0.721377432346344f, -0.29935362935066223f, -0.49368005990982056f, -1.5796416997909546f, 0.3189202845096588f, -0.5672794580459595f, 1.5067633390426636f, 1.161847710609436f, -0.08308011293411255f, -2.262038469314575f, 0.8626111745834351f, 0.196146160364151f, 0.2249874323606491f, 0.6830949187278748f, 0.0019485306693241f, 1.01323664188385f, -0.6184468865394592f, 0.7670295834541321f, 2.8818862438201904f, 0.6386357545852661f, -0.10977188497781754f, -0.17038141191005707f, -0.938051164150238f, -0.11422315984964371f, 0.1034913882613182f, -1.2966164350509644f, -0.41023650765419006f, -0.8332573175430298f, -0.9556332230567932f, 0.1063748151063919f, -0.20979070663452148f, 0.6346396803855896f, 0.545662522315979f, -0.937210738658905f, -1.1672022342681885f, -0.4361383616924286f, 0.0964970588684082f, -0.3027929663658142f, -0.638506293296814f, -0.05629456788301468f, 0.11661393195390701f, 0.3269132375717163f, -0.7588294148445129f, -0.952683687210083f, -0.7779799103736877f, 0.3503226041793823f, 1.2010222673416138f, 0.8768793344497681f, -0.7691330313682556f, -2.330476999282837f, 1.9413354396820068f, -1.0022987127304077f, -1.4932167530059814f, -1.2868070602416992f, 0.21946865320205688f, 0.8497498631477356f, -0.02522038109600544f, 0.6064606308937073f, 1.6981608867645264f, -0.36366915702819824f, 0.6935068964958191f, 0.12109924107789993f, 0.675596296787262f, 0.8142848610877991f, 0.0793362557888031f, -0.7808834314346313f, -1.163609266281128f, 0.3780289888381958f, 0.5237964987754822f, -1.3662828207015991f, -0.439242959022522f, -0.8599120378494263f, 0.3788713812828064f, -0.46068817377090454f, 0.7436892986297607f, -1.2479252815246582f, -0.4358613193035126f, 0.2984321117401123f, -1.8900177478790283f, 0.19737061858177185f, -0.7584667801856995f, 0.8250064849853516f, -1.5966553688049316f, 0.04276036471128464f, 0.7530690431594849f, -0.04625223949551582f, -0.1391070932149887f, -0.6105663776397705f, 0.24455690383911133f, -0.8800807595252991f, -0.03276701644062996f, -0.09261447191238403f, -0.9824548959732056f, 1.3447729349136353f, 0.19431965053081512f, -0.6367602348327637f, 0.941990315914154f, 1.2206652164459229f, 0.08320564776659012f, 0.5858650803565979f, -0.2219216674566269f, -0.6149584650993347f, -0.018692156299948692f, 1.0629346370697021f, -0.8915789723396301f, -1.4766337871551514f, 1.0600273609161377f, 0.6533640027046204f, -0.9062879085540771f, 1.4614392518997192f, 1.6020095348358154f, -0.8768263459205627f, 0.43399056792259216f, -0.06297583878040314f, -0.9222075343132019f, -1.3687634468078613f, 0.5819101333618164f, 1.2366944551467896f, 1.6560721397399902f, 1.2221823930740356f, 1.4343817234039307f, 0.982934296131134f, 0.003909424878656864f, 0.16369947791099548f, 1.25956130027771f, 1.0566132068634033f, -1.0371782779693604f, -1.5509816408157349f, 1.1234275102615356f, 1.1463638544082642f, -0.3711417317390442f, 1.1168509721755981f, -1.9369032382965088f, -0.8143561482429504f, 1.0661178827285767f, 0.771891176700592f, -1.9291707277297974f, 0.034095875918865204f, -0.6951659917831421f, 1.3456140756607056f, 0.34626761078834534f, 0.8155362606048584f, -0.7486876249313354f, -0.7881835699081421f, 0.5057575702667236f, 1.2215378284454346f, -0.9714295268058777f, 0.27692535519599915f, -1.5334482192993164f, 1.146209955215454f, -0.18189188838005066f, -1.2797164916992188f, -0.5713837146759033f, 1.1689343452453613f, -1.8829604387283325f, -0.369537353515625f, -2.0042972564697266f, -0.8240620493888855f, -0.3844030499458313f, 0.5684375166893005f, -1.137097716331482f, -0.31372663378715515f, 0.15282166004180908f, 1.0580254793167114f, -0.17965702712535858f, -1.178733229637146f, -1.2379671335220337f, -1.5433486700057983f, -0.6671661734580994f, 0.9025653600692749f, 0.304572194814682f, -0.47141557931900024f, 0.005512263625860214f, 0.8932762742042542f, 0.14350247383117676f, 0.7622402906417847f, -0.758182168006897f, -0.7234253287315369f, 0.7250988483428955f, -0.05147926136851311f, -0.6577823162078857f, -0.036672789603471756f, 0.514220118522644f, -0.6020243763923645f, -0.09203547239303589f, -1.7167327404022217f, -0.7002271413803101f, 0.4779217541217804f, 0.46033239364624023f, -0.386881560087204f, -0.0812682956457138f, 0.5568539500236511f, 0.04228341206908226f, -0.3627345860004425f, -1.3895257711410522f, 0.17128385603427887f, -1.2317392826080322f, 0.6718097925186157f, -0.15104348957538605f, 0.3213997781276703f, 0.9548746347427368f, 0.04617787152528763f, 1.7811017036437988f, 0.8509474396705627f, 1.179113507270813f, -0.35717684030532837f, 1.1261653900146484f, 1.2983627319335938f, -0.6900910139083862f, 0.4424906373023987f, 1.1874427795410156f, -0.21165163815021515f, -1.2463988065719604f, 0.9242297410964966f, 0.13773603737354279f, -0.060615699738264084f, 0.18482893705368042f, 0.09389957040548325f, -0.2936820983886719f, 0.36884430050849915f, -1.0105520486831665f, -1.1568186283111572f, 0.04061844199895859f, 1.7561308145523071f, -0.5621538162231445f, 0.19418247044086456f, 0.027558235451579094f, -0.1787608563899994f, -1.4996418952941895f, 0.26971569657325745f, 1.2250828742980957f, -1.170371651649475f, -0.11677189916372299f, -1.35635244846344f, 0.5242968797683716f, 1.0622302293777466f, -0.620258629322052f, -1.7659958600997925f, -0.06464217603206635f, 0.21129246056079865f, 0.825295627117157f, -0.4999869465827942f, 2.0010547637939453f, -0.8498311638832092f, -0.5403968691825867f, 0.628537118434906f, -0.7642602920532227f, -2.773613929748535f, -0.22911065816879272f, -1.0603324174880981f, 1.0186315774917603f, 1.763922929763794f, -1.4548780918121338f, -2.5379507541656494f, 3.445625066757202f, 1.4860185384750366f, 2.268347978591919f, -0.5500510334968567f, -0.12460697442293167f, 1.166976809501648f, 1.2409895658493042f, -0.7066188454627991f, 0.9183834791183472f, -0.02982438914477825f, -0.3448553681373596f, 1.0654057264328003f, -0.15129554271697998f, -0.048379626125097275f, -0.5897568464279175f, 0.09694309532642365f, -1.1129159927368164f, 0.5585124492645264f, -1.7732727527618408f, 0.023851748555898666f, -1.156516194343567f, -0.34847185015678406f, -1.6212294101715088f, -0.8558416366577148f, 0.40576910972595215f, -0.91661536693573f, -1.0765177011489868f, 0.5447612404823303f, 1.1705763339996338f, 0.9127081632614136f, -0.6960206627845764f, -0.7912065982818604f, -0.4341987073421478f, 0.3491472005844116f, -0.9907275438308716f, -1.5203826427459717f, 0.9009959697723389f, 0.07418181002140045f, -0.9035959839820862f, 0.9040606021881104f, 0.25104597210884094f, 0.4702746272087097f, -0.5477657318115234f, 1.522364616394043f, 1.6411151885986328f, -1.3906173706054688f, 0.5155009627342224f, 0.37119555473327637f, -0.015211731195449829f, 0.8483667373657227f, 0.634861409664154f, 0.5293999314308167f, -0.18842656910419464f, 1.757521390914917f, 1.3967915773391724f, -0.5659265518188477f, 0.1281542032957077f, 1.8777929544448853f, -0.3289257884025574f, 0.02351505309343338f, 0.428303986787796f, -0.7096118927001953f, 0.723761260509491f, 1.5117958784103394f, 0.5427215099334717f, 2.0354838371276855f, 1.0238486528396606f, -0.39082857966423035f, -1.2726235389709473f, -0.6254484057426453f, 0.9178156852722168f, 0.3840969204902649f, -0.19802889227867126f, 0.5223742127418518f, 0.05841278284788132f, 0.28558799624443054f, -1.0703928470611572f, 0.9307976365089417f, -0.18846826255321503f, -0.8914596438407898f, 0.12508316338062286f, 0.3772087097167969f, 0.9574744701385498f, -1.9291036128997803f, 0.5764585137367249f, 0.20935998857021332f, 0.5120382905006409f, -0.7815778851509094f, -1.478035569190979f, 0.051756951957941055f, 2.1100034713745117f, 0.020343216136097908f, -0.6159464716911316f, -0.4243218004703522f, -0.5821011662483215f, 0.5247467160224915f, -0.8761705756187439f, 0.48434850573539734f, 0.019129564985632896f, 0.03497670218348503f, -0.35594069957733154f, 1.505028486251831f, -0.8405776023864746f, -0.15094131231307983f, -0.5910253524780273f, 0.4260155260562897f, 0.7726138234138489f, 0.9289091229438782f, -1.0194525718688965f, -0.4821736812591553f, -0.4351550042629242f, 0.3517809212207794f, 1.2222594022750854f, -0.030398759990930557f, -1.0700304508209229f, -0.4833625555038452f, -0.5188538432121277f, -1.1346086263656616f, -1.0445209741592407f, 0.4199317395687103f, 0.8357481956481934f, 0.9475129246711731f, -0.34265488386154175f, -0.9238923788070679f, 0.3848164975643158f, 0.4773769974708557f, -1.2878144979476929f, 0.2491273283958435f, 0.32681646943092346f, 1.0925219058990479f, -0.7518913149833679f, -1.2260730266571045f, -0.3927396237850189f, -0.7111839056015015f, 0.5547724366188049f, 1.4059346914291382f, -0.22924941778182983f, 0.4364280700683594f, 1.2370060682296753f, 1.1473842859268188f, 0.29434895515441895f, -0.1789979487657547f, 1.9290111064910889f, 1.1900452375411987f, -0.10487240552902222f, -0.5064011812210083f, 2.4400694370269775f, -0.33963847160339355f, 1.7511601448059082f, 1.3401062488555908f, -0.9492565989494324f, -0.8270853161811829f, -1.2403943538665771f, -0.2053070068359375f, -0.2497664988040924f, -0.7054344415664673f, -1.7715047597885132f, 0.4937354028224945f, -0.17678959667682648f, 1.410936951637268f, 2.476804733276367f, -0.8441710472106934f, -1.912861704826355f, 0.007326157763600349f, 0.19589348137378693f, 1.5108956098556519f, 1.0457415580749512f, 1.560778021812439f, 0.24919238686561584f, -1.4739270210266113f, 0.22397132217884064f, 1.1544770002365112f, 0.4349344074726105f, 0.5612083077430725f, 0.6398148536682129f, 0.7980097532272339f, 0.5769069790840149f, -0.22143854200839996f, 0.31076598167419434f, -0.052180398255586624f, 1.7757917642593384f, -1.198473572731018f, -1.1297553777694702f, -1.0952261686325073f, -1.1415528059005737f, 0.5066304802894592f, -0.6875526905059814f, -1.7462857961654663f, 0.06312361359596252f, -1.7986558675765991f, 0.6912683248519897f, 0.49413225054740906f, 0.14730526506900787f, 2.274590015411377f, -1.4762635231018066f, 1.167490005493164f, -0.18522614240646362f, 0.37665531039237976f, -2.800556182861328f, 1.1855055093765259f, 0.7821402549743652f, 0.2779245674610138f, -0.489844411611557f, 0.7103408575057983f, -0.6894504427909851f, -0.6078459620475769f, 0.5198960900306702f, -0.18495284020900726f, -1.3136197328567505f, -0.783242404460907f, -0.47829392552375793f, 1.730208158493042f, 1.0595130920410156f, -0.251127690076828f, 1.2796905040740967f, 1.1880545616149902f, 0.010817415080964565f, 1.351500153541565f, 0.399687796831131f, -0.10175959765911102f, -0.2829807698726654f, -0.5876336097717285f, -0.7546091079711914f, -1.0380902290344238f, -0.6061143279075623f, -0.26778313517570496f, -0.11727923899888992f, 2.056168794631958f, 1.2978315353393555f, -0.527733564376831f, 0.2199985533952713f, 2.055619716644287f, 0.3816293179988861f, -0.8949641585350037f, 0.6981958150863647f, -0.9663392901420593f, 0.6600029468536377f, -0.3451770842075348f, -0.033411234617233276f, -0.10130416601896286f, 0.6449755430221558f, 1.5684525966644287f, -0.630508303642273f, -0.23472319543361664f, 0.3285580277442932f, -0.47698524594306946f, 0.3249095380306244f, 0.6602651476860046f, -1.652677059173584f, -1.596110463142395f, 0.05211031809449196f, -1.4231536388397217f, -0.16193436086177826f, -0.7287773489952087f, -0.8276495337486267f, -0.9271824359893799f, -0.24771355092525482f, -0.283514142036438f, -1.265464186668396f, -1.345624327659607f, 0.14869070053100586f, -0.5625298023223877f, 1.319007396697998f, -1.1078379154205322f, -0.38139888644218445f, 0.673722505569458f, 0.2882064878940582f, -0.7846477627754211f, 0.886084258556366f, -1.4957531690597534f, -0.3524166941642761f, -0.6002296209335327f, -0.5345580577850342f, -1.4335700273513794f, -0.06124546006321907f, -1.0124340057373047f, 0.7134581804275513f, -0.3809734880924225f, 1.6037938594818115f, -0.6744462847709656f, 0.03632983937859535f, 0.2812545895576477f, 2.2968475818634033f, 1.8312735557556152f, 0.0294801015406847f, 0.7718657851219177f, -0.6380594372749329f, -0.05797451362013817f, -1.4051520824432373f, -0.8756160736083984f, -0.9422666430473328f, -0.14667245745658875f, -1.9506268501281738f, -1.632828712463379f, 0.7152358293533325f, -1.8892930746078491f, 2.4672696590423584f, 0.08700139075517654f, 1.649484395980835f, -0.6159215569496155f, -0.5199353098869324f, 0.01673818565905094f, 0.5336771607398987f, 0.2974875271320343f, 1.8911962509155273f, 0.009384111501276493f, 1.4318883419036865f, -0.49658793210983276f, -0.6031076312065125f, -0.32575365900993347f, 1.9036880731582642f, -1.842443585395813f, -0.16547387838363647f, -0.25706610083580017f, 1.3179066181182861f, -0.6072822213172913f, -0.16537998616695404f, 0.6803566813468933f, 0.1680675745010376f, 1.6327744722366333f, 1.3595441579818726f, 0.6304294466972351f, 0.13153165578842163f, 0.36382463574409485f, -0.7942026853561401f, 2.110002040863037f, -1.1197839975357056f, 0.13227719068527222f, -0.3069077730178833f, 2.2180261611938477f, 0.7556464672088623f, -2.05973744392395f, -0.27732527256011963f, -0.12983323633670807f, -1.0805774927139282f, -1.4954025745391846f, -0.8487699627876282f, 0.8632985949516296f, -0.18222470581531525f, 0.3396982252597809f, 0.023615580052137375f, -0.5582384467124939f, -2.211786985397339f, -0.792870283126831f, 1.4188532829284668f, 1.240154504776001f, 1.2471561431884766f, 1.5289140939712524f, -0.24465647339820862f, 0.09731336683034897f, 0.8652670383453369f, -0.29969948530197144f, -1.841177225112915f, 1.1419644355773926f, 0.3660371005535126f, 1.273138165473938f, -0.7165514230728149f, 0.3878994286060333f, -0.5754631757736206f, 1.2297093868255615f, -0.4566229581832886f, -0.6573424935340881f, 0.7881364822387695f, 0.3378683924674988f, -1.988023042678833f, 0.7956935167312622f, 0.9823195338249207f, -0.4144572615623474f, 1.3572540283203125f, 0.3109254240989685f, -0.6753576993942261f, -1.5371414422988892f, 0.9313072562217712f, -2.2768142223358154f, 0.4319401681423187f, 0.07773400843143463f, -1.5976078510284424f, 1.848445177078247f, 1.5493229627609253f, 0.1915358453989029f, -1.266395926475525f, -2.1607372760772705f, 0.7240417003631592f, -0.7832543253898621f, 1.1483440399169922f, -0.10642467439174652f, -0.32092228531837463f, 1.0402002334594727f, 0.11275313794612885f, 0.37306782603263855f, 0.8337973356246948f, 1.8036106824874878f, 0.7735506892204285f, -1.1966158151626587f, -0.6088662147521973f, 0.16352267563343048f, -0.3072216808795929f, -0.5694004893302917f, 0.13301636278629303f, -1.508280873298645f, -0.8582409024238586f, -0.7953895330429077f, -1.2091906070709229f, -0.7641999125480652f, 0.2154434323310852f, 0.2061154842376709f, 0.3153077960014343f, -0.3388381898403168f, -0.6360014081001282f, -0.45390239357948303f, -2.7026309967041016f, 0.6710167527198792f, 0.8398252725601196f, -2.002260208129883f, -0.6277503967285156f, -1.3664311170578003f, -0.47482022643089294f, -1.1213213205337524f, 0.07767737656831741f, -0.44291654229164124f, -1.2841796875f, -1.0159707069396973f, 1.3361929655075073f, -0.46696820855140686f, -0.2509534955024719f, 1.4244478940963745f, -0.6108700037002563f, -0.4096096158027649f, -0.4688802659511566f, -1.2304245233535767f, -0.14591790735721588f, 0.9639950394630432f, -0.9581076502799988f, 0.8579769134521484f, 0.17619068920612335f, -0.009090420790016651f, 2.2291040420532227f, -0.599851667881012f, 0.5864706635475159f, -0.40187644958496094f, 0.7005379796028137f, 2.269195318222046f, 1.1897660493850708f, -0.530225396156311f, 0.22658583521842957f, 0.8770406246185303f, -0.42359691858291626f, 0.04045804589986801f, -0.3348093032836914f, -1.6933249235153198f, 0.578863799571991f, 0.253021240234375f, 0.8249357342720032f, 0.17797666788101196f, 0.21032211184501648f, -1.3109723329544067f, 1.4387873411178589f, 1.3104835748672485f, -0.4802001118659973f, 0.2532861530780792f, 0.34186819195747375f, -2.0921499729156494f, -1.4881683588027954f, 0.24693161249160767f, -1.7462626695632935f, 0.28935855627059937f, 1.5511679649353027f, 0.8895691633224487f, 0.4553866684436798f, -1.143309235572815f, -0.06552600860595703f, 0.03079218976199627f, -1.0684497356414795f, -0.3178943395614624f, 2.2536263465881348f, -0.19900000095367432f, 0.5933657884597778f, 1.5937228202819824f, 0.8582064509391785f, 0.11251302808523178f, 0.2912493050098419f, 0.12979160249233246f, 0.08444352447986603f, 2.015190601348877f, 0.5599141120910645f, 1.3394620418548584f, 0.6110420823097229f, -0.5921895503997803f, -0.16634242236614227f, -0.3773656487464905f, 1.1717573404312134f, 0.6101416945457458f, 1.917343020439148f, 2.563725471496582f, 3.035121202468872f, -0.534939169883728f, -0.4571112394332886f, 0.802577018737793f, -0.28734639286994934f, 0.12288182973861694f, 0.21209175884723663f, -0.996752142906189f, 0.6648997068405151f, -1.1771180629730225f, 0.517611563205719f, 2.260444164276123f, 0.8793008327484131f, -1.4391361474990845f, -0.47865501046180725f, -0.12678508460521698f, -1.1487538814544678f, 0.9171100854873657f, 0.9602430462837219f, 0.20192822813987732f, -0.6621134877204895f, 0.283156156539917f, 1.1861251592636108f, -0.6021904945373535f, -0.2499617040157318f, 1.7409440279006958f, -0.7325262427330017f, -0.3309515118598938f, -0.7796709537506104f, 1.6620672941207886f, -0.0578240342438221f, 0.2314254492521286f, 0.22709836065769196f, 2.065865993499756f, 0.4171837866306305f, 0.3846955895423889f, -0.01396828331053257f, 0.5790372490882874f, 0.2881997525691986f, 0.8778835535049438f, 0.5851041674613953f, -0.29608187079429626f, 0.33585336804389954f, -0.7194162011146545f, -0.07809803634881973f, 0.35557520389556885f, -1.7239394187927246f, 0.7924410104751587f, 0.030582552775740623f, 1.0178154706954956f, 1.6657739877700806f, 0.17066167294979095f, -0.2777872681617737f, -0.19989891350269318f, 1.6998308897018433f, 0.7318398952484131f, 1.2774115800857544f, -0.3473694622516632f, -0.7603669166564941f, 1.2199066877365112f, -0.3723330795764923f, -1.8119957447052002f, -0.9909027814865112f, -0.3076462149620056f, 0.01513738464564085f, -1.9378867149353027f, -1.8149548768997192f, 1.3613444566726685f, -0.7856798768043518f, -2.4690840244293213f, 2.432499408721924f, 0.27381452918052673f, 0.8801257610321045f, -0.49671268463134766f, 0.056602153927087784f, 1.435631513595581f, 0.36377331614494324f, 0.46456536650657654f, 1.9551998376846313f, 0.6760215759277344f, 1.1773208379745483f, -0.5477144718170166f, -0.48883718252182007f, -0.2574329078197479f, 0.46062397956848145f, -1.6101100444793701f, 0.2585121691226959f, 1.7188366651535034f, 0.5200220942497253f, -1.301006555557251f, -1.5038570165634155f, -0.31398269534111023f, 1.2563191652297974f, -0.5480050444602966f, -0.06532658636569977f, 2.6805763244628906f, -0.9649611115455627f, -0.18265697360038757f, 0.3190588653087616f, 0.006673657335340977f, 0.566218376159668f, 1.7436981201171875f, 1.3330131769180298f, 1.3450311422348022f, -1.0255969762802124f, 1.3098556995391846f, -0.4485261142253876f, 0.8979019522666931f, -0.1220834031701088f, -1.0596239566802979f, 0.14630116522312164f, -0.870776891708374f, -0.24674776196479797f, -1.2373387813568115f, 0.8207234740257263f, -0.5435441136360168f, 1.1541569232940674f, -0.38771313428878784f, 0.12649448215961456f, -1.2653104066848755f, 1.7627143859863281f, -0.2666304111480713f, 0.5257315635681152f, 0.6358874440193176f, 0.10120877623558044f, 0.17400647699832916f, 1.1357200145721436f, 0.036105670034885406f, -0.6797627210617065f, -0.16857317090034485f, 0.26855215430259705f, -1.4135205745697021f, -2.3971786499023438f, 0.7723278403282166f, -0.15770527720451355f, 0.45117413997650146f, -2.815762519836426f, 0.19697873294353485f, 0.2619018256664276f, -0.8475749492645264f, 0.47121289372444153f, 0.3821605145931244f, -0.2688539922237396f, 1.0989755392074585f, -1.0098387002944946f, 0.688886284828186f, 0.5349103808403015f, 2.787760019302368f, 0.38038408756256104f, 0.22671592235565186f, 0.5783246755599976f, 0.6597211360931396f, -0.28375279903411865f, -0.6992143392562866f, 0.7167068123817444f, -0.09313177317380905f, 0.6840168237686157f, -0.19578158855438232f, -0.9126741886138916f, -0.2800213098526001f, -0.3882693350315094f, 0.10380307585000992f, -1.9663856029510498f, -1.013744592666626f, -1.0242971181869507f, 0.8466978073120117f, 1.2233283519744873f, -0.6524525284767151f, -0.6867133378982544f, 1.1395643949508667f, -0.696540355682373f, -0.39359593391418457f, 0.5099198222160339f, -0.15132363140583038f, 0.6866339445114136f, 0.5310931205749512f, -1.3795819282531738f, -0.6988955140113831f, 0.3475550413131714f, 1.001903772354126f, 0.7994512319564819f, 0.8503185510635376f, 0.791765034198761f, -0.8879274725914001f, 1.02428138256073f, 0.1911749392747879f, 0.8436497449874878f, -0.02484050951898098f, -0.7801733613014221f, 0.625577986240387f, 1.5644149780273438f, 0.5320438146591187f, 1.0700255632400513f, -0.5013832449913025f, -0.3839770257472992f, 0.8672234416007996f, 0.8700053095817566f, 1.8044092655181885f, 0.5074418187141418f, -1.0803741216659546f, 1.7308460474014282f, -0.009462441317737103f, 1.9249420166015625f, -0.3632628321647644f, 0.6629142165184021f, -0.6219039559364319f, 1.013214111328125f, -1.5853021144866943f, -0.9035424590110779f, -0.1714278757572174f, 0.9996393918991089f, 0.2449631243944168f, 0.07007400691509247f, 1.4646685123443604f, 0.4391697347164154f, 1.424518346786499f, -0.32468941807746887f, 0.35460788011550903f, -0.3405316174030304f, -0.6941221952438354f, 0.6556671857833862f, -1.087321400642395f, -1.1486494541168213f, 2.4220407009124756f, 0.7684311866760254f, -1.5410068035125732f, -0.411981463432312f, 1.3056882619857788f, -0.338508278131485f, 0.2913444936275482f, -0.8357943892478943f, -1.1213253736495972f, 0.3143851161003113f, -0.42382940649986267f, -0.43293496966362f, -0.9816297292709351f, 0.058465536683797836f, -1.3252078294754028f, -0.7915613055229187f, 0.47723037004470825f, 0.43918681144714355f, -0.3302071988582611f, 1.7920981645584106f, -1.0610337257385254f, -0.09547755867242813f, -1.1624996662139893f, -0.6900646686553955f, 0.3918372392654419f, -0.0669134333729744f, 1.0711755752563477f, 1.3083903789520264f, -0.05556102097034454f, 0.7882350087165833f, 0.37722504138946533f, -0.3213580846786499f, 0.5956750512123108f, -0.5032515525817871f, -0.7394251227378845f, -0.7672699093818665f, -0.040719106793403625f, 0.15840405225753784f, -0.4783642590045929f, 0.4646506905555725f, -0.49036842584609985f, -0.6546841263771057f, 2.7124791145324707f, 0.4293099343776703f, -1.0469098091125488f, -1.0857888460159302f, -0.05841507390141487f, 0.5456286668777466f, 0.2792662978172302f, 2.1168277263641357f, -1.5103271007537842f, -1.307977557182312f, -1.9681334495544434f, -0.8398545384407043f, 1.3870203495025635f, 0.9826740026473999f, -0.2890629470348358f, 1.0335131883621216f, -0.19352838397026062f, 0.07122802734375f, -0.16152557730674744f, 1.0519514083862305f, -1.4766151905059814f, -1.2671267986297607f, 0.2393292933702469f, 1.2190254926681519f, 0.17837846279144287f, -1.3159326314926147f, 0.646186888217926f, 1.6116095781326294f, -0.05114508792757988f, 0.41359904408454895f, -0.20463405549526215f, -2.1418421268463135f, 1.7502615451812744f, -1.4018466472625732f, -0.5717736482620239f, -0.38931718468666077f, -0.9860185384750366f, 0.5779818296432495f, 0.4738202393054962f, -0.7853313088417053f, -0.1793491244316101f, -0.1285407692193985f, 0.9474418759346008f, -0.07980115711688995f, -0.6013848185539246f, -0.5239112377166748f, -0.1949448138475418f, 0.7724379897117615f, -0.11171314865350723f, 0.5611289739608765f, -1.5560880899429321f, 1.4754453897476196f, 1.4865750074386597f, -0.02104124240577221f, 0.031055578961968422f, 1.0900859832763672f, -0.5583686828613281f, 1.3067461252212524f, -0.9643430113792419f, 1.6326349973678589f, -1.144444465637207f, 0.13422854244709015f, 1.668276309967041f, -0.6358500719070435f, -0.8220173120498657f, 1.151316523551941f, -0.9632699489593506f, -0.17086558043956757f, 0.14713206887245178f, -0.13801060616970062f, -0.1489359736442566f, -0.06645472347736359f, -0.880412757396698f, 0.1913401335477829f, -0.05544678866863251f, -0.2204407900571823f, -1.1233654022216797f, -1.1245696544647217f, 0.674127995967865f, 0.2520127594470978f, 0.7975156307220459f, 0.698857843875885f, -0.4366423487663269f, -2.0883853435516357f, -1.366032361984253f, 0.09937097132205963f, -0.3652037978172302f, 1.2572640180587769f, -0.8791263699531555f, 1.3767470121383667f, 0.6487414836883545f, -0.22251465916633606f, -0.7945832014083862f, 1.3094075918197632f, 1.1708104610443115f, 0.9832903742790222f, -0.7685240507125854f, -0.5897563099861145f, -0.008548072539269924f, 0.7963455319404602f, -0.6708536148071289f, -0.16284485161304474f, -1.8156386613845825f, 0.1582425981760025f, 1.5490630865097046f, 0.6892044544219971f, -1.5244767665863037f, 0.1314752995967865f, -0.5694751739501953f, -0.07285541296005249f, 0.02161409705877304f, 0.27252840995788574f, 1.5375657081604004f, -0.16460900008678436f, 0.04459971562027931f, 0.4962165355682373f, 0.9521092176437378f, 0.18977606296539307f, 1.1129406690597534f, -1.742975115776062f, 0.015372429974377155f, 0.43921586871147156f, -0.4629494845867157f, 0.683012068271637f, 0.0549420528113842f, 1.1227099895477295f, -0.9156265258789062f, 0.7202309370040894f, -1.7771120071411133f, -0.49309614300727844f, -0.3537367880344391f, 0.6029451489448547f, 1.4749457836151123f, -1.2571945190429688f, 1.1716068983078003f, -1.2938947677612305f, -0.027610722929239273f, -0.04813985526561737f, -1.6245776414871216f, -0.1380276381969452f, -0.7646363377571106f, -0.24534782767295837f, 0.21846671402454376f, 0.8769089579582214f, -1.0646121501922607f, 0.50408536195755f, 0.15752233564853668f, -0.5226418375968933f, -1.4756296873092651f, 0.25705486536026f, -1.7179490327835083f, 1.3074569702148438f, -0.4480331838130951f, 1.1221709251403809f, 1.4023377895355225f, 1.4818451404571533f, -1.4184188842773438f, 0.2722645699977875f, 1.2556324005126953f, -1.6670103073120117f, 1.050782322883606f, 0.13771948218345642f, -1.1567072868347168f, 1.0360978841781616f, -0.8660301566123962f, -1.0625808238983154f, 1.0239890813827515f, 0.7085686922073364f, 1.8764188289642334f, 2.794304609298706f, 0.38867735862731934f, 1.6592570543289185f, -0.8973153829574585f, -0.428587943315506f, -0.8077448010444641f, -2.2226719856262207f, 1.3841030597686768f, 0.27507975697517395f, 1.8155642747879028f, 0.5318379998207092f, 1.278135895729065f, -0.632580041885376f, -1.036582350730896f, 0.294900506734848f, -0.8918796181678772f, -0.08464081585407257f, 0.7254555821418762f, 0.40998774766921997f, -0.0699654296040535f, 0.8239985108375549f, 2.0619850158691406f, -1.0847980976104736f, -1.5027097463607788f, 0.4682971239089966f, -2.0921216011047363f, -0.31479185819625854f, 0.35259687900543213f, -0.6293240189552307f, -1.997800350189209f, -0.6937687397003174f, 0.43931636214256287f, -0.2434539496898651f, -1.389511227607727f, -0.0009331775945611298f, 0.4944293797016144f, -1.3989880084991455f, 0.3048802316188812f, 0.612401008605957f, -1.0865437984466553f, -0.7029951810836792f, -0.651735246181488f, 0.02101830020546913f, -0.07500791549682617f, -1.676796793937683f, 1.5087897777557373f, -0.8013211488723755f, -1.030982494354248f, -1.4118940830230713f, -0.8552696704864502f, -0.1083751767873764f, 2.1871159076690674f, -0.6815629601478577f, -1.4610782861709595f, -0.5785004496574402f, 2.1495704650878906f, -0.179605171084404f, 1.1426492929458618f, -0.054547298699617386f, 0.40587425231933594f, 0.6724022030830383f, 0.19652637839317322f, -0.07756809890270233f, -1.4263975620269775f, -1.0503880977630615f, -1.5335310697555542f, 0.4794066250324249f, 1.1617707014083862f, 0.8538435697555542f, -0.20266015827655792f, -0.725740909576416f, -0.9262224435806274f, 0.8973854184150696f, -0.7538284063339233f, -0.8116031885147095f, 0.5351168513298035f, 1.988852858543396f, 1.0685486793518066f, -0.5014570951461792f, 0.7195857763290405f, 1.110618233680725f, 0.1288852095603943f, -1.2922874689102173f, 0.6346917748451233f, 1.5678290128707886f, 0.03146770969033241f, 1.8645329475402832f, -0.8617974519729614f, 0.051693860441446304f, -1.4663174152374268f, -0.26106584072113037f, -0.06878392398357391f, 0.8156912922859192f, 0.4850883483886719f, -2.2269856929779053f, 0.5959717631340027f, 0.21369199454784393f, 0.2891088128089905f, 0.3474847674369812f, -0.43252453207969666f, -0.5997704267501831f, 0.2910449504852295f, -0.8439115881919861f, -0.013285640627145767f, -0.5315346121788025f, 0.08023621141910553f, -0.6925831437110901f, -0.6154842972755432f, -1.4682725667953491f, 0.005962289869785309f, -0.17264695465564728f, -1.3461240530014038f, 0.5067698955535889f, -0.7377328872680664f, -0.006388451438397169f, -1.411285400390625f, 0.23984402418136597f, 0.4191838204860687f, 1.395574688911438f, 0.976121723651886f, 0.40693262219429016f, -0.6316997408866882f, 1.5522570610046387f, 0.2695809602737427f, 1.6630321741104126f, 1.0006709098815918f, -0.6625555157661438f, -0.05369861051440239f, 0.2828640341758728f, 0.8807763457298279f, -0.3764995038509369f, 1.8677223920822144f, 2.1043002605438232f, -0.8154667019844055f, -0.4016047418117523f, -0.07730520516633987f, 0.4082213044166565f, -0.7410086393356323f, -2.3087081909179688f, -0.031602293252944946f, -1.454459547996521f, 0.7048721313476562f, -0.5495360493659973f, -1.0822854042053223f, -1.1814165115356445f, -0.7662639021873474f, 2.26790452003479f, -0.9821573495864868f, -0.7700114250183105f, -0.6950612664222717f, -0.47602152824401855f, -2.168846368789673f, -0.4960733652114868f, 1.8063979148864746f, -2.195838451385498f, -1.2756946086883545f, -0.23146040737628937f, -0.6978404521942139f, 0.05868193879723549f, 0.3181943893432617f, -0.9001671671867371f, 1.4585676193237305f, 0.9991294741630554f, -0.958099365234375f, -1.5130436420440674f, 0.8987486362457275f, 0.6024074554443359f, 1.213675856590271f, -0.9290968179702759f, 0.9378061294555664f, 0.32025381922721863f, -0.6372569799423218f, 0.5549997091293335f, 0.4728609323501587f, 1.5382329225540161f, -2.230900526046753f, 2.1022186279296875f, 1.7723872661590576f, -0.29818224906921387f, 1.774613618850708f, -0.403571754693985f, -0.763476550579071f, -0.13895398378372192f, -1.8085858821868896f, -0.1992855966091156f, -0.38452786207199097f, 0.772721529006958f, -0.761553168296814f, 0.32450324296951294f, -0.6566532254219055f, 1.0444575548171997f, -1.0310343503952026f, 0.13313262164592743f, 0.32965973019599915f, -0.7564722895622253f, 1.5161906480789185f, -0.442477822303772f, -0.19663105905056f, -0.5199072957038879f, 0.19425715506076813f, 0.468250036239624f, 0.658501386642456f, -0.16666415333747864f, -0.4670298099517822f, 1.4936802387237549f, -0.8678151965141296f, -0.26301294565200806f, 0.20588766038417816f, 0.022425031289458275f, 0.6587340831756592f, -0.20476730167865753f, -1.0465848445892334f, -1.2654074430465698f, -0.3633078634738922f, -0.4297706186771393f, 0.6679531335830688f, 1.0863715410232544f, 0.7616772055625916f, -0.011849306523799896f, -1.2028323411941528f, 0.5853366255760193f, -0.10431576520204544f, 0.2191448211669922f, -0.8329556584358215f, -0.2799490988254547f, 0.6428387761116028f, 1.405016303062439f, 1.3759163618087769f, 1.4840714931488037f, -1.5530608892440796f, -0.9330264925956726f, -1.1588698625564575f, -0.48916497826576233f, -0.6450856328010559f, -0.11284743994474411f, -2.4587764739990234f, 0.7599589824676514f, 0.9755558371543884f, 0.05122251436114311f, 0.960921049118042f, -0.6847841143608093f, 0.40178006887435913f, -0.9434769749641418f, -0.23219886422157288f, -0.2043316662311554f, -0.8605437278747559f, -0.3255579173564911f, -0.7162472605705261f, -0.08609210699796677f, -3.6307897567749023f, -0.6838181614875793f, -0.489888072013855f, -1.0136138200759888f, -0.8829219937324524f, 1.1272135972976685f, -1.2231221199035645f, -0.7950332760810852f, 0.6623095273971558f, 2.028413772583008f, 0.8947616219520569f, -1.1885942220687866f, -0.400406152009964f, 0.9303890466690063f, -1.0271012783050537f, 0.60735023021698f, -2.186370372772217f, -1.2513858079910278f, -1.5937029123306274f, -1.391968846321106f, -0.7062578201293945f, 0.5445007681846619f, 0.0782814472913742f, -1.7333787679672241f, -0.8157885074615479f, 0.3931215703487396f, -0.11953603476285934f, 0.14065788686275482f, 1.6809849739074707f, -0.2840443551540375f, -1.4785282611846924f, 0.227834552526474f, 0.26440107822418213f, -0.0759536400437355f, 0.9481480121612549f, 0.8856539726257324f, -1.2799997329711914f, -0.2185828685760498f, -0.15562321245670319f, 1.1858826875686646f, 1.7509973049163818f, 0.4133126139640808f, 0.7854304909706116f, 0.64668208360672f, -0.11213088780641556f, 0.8463886380195618f, 0.045808374881744385f, -0.6186387538909912f, -0.5598823428153992f, 0.3789231479167938f, -0.4264816343784332f, 0.9161636829376221f, 0.13592901825904846f, 0.4121098220348358f, 0.40913498401641846f, 0.4465799629688263f, -0.07119281589984894f, 0.4638987183570862f, -0.5617773532867432f, -0.23070071637630463f, -0.5451535582542419f, 0.018565375357866287f, -0.10694064944982529f, 1.1308897733688354f, 1.4536970853805542f, 0.6201399564743042f, -1.4827039241790771f, 0.4850791096687317f, -0.28107115626335144f, -1.132474660873413f, 0.4989188015460968f, 1.1301037073135376f, -0.3190559148788452f, -0.02982708252966404f, -0.4788385033607483f, 0.3957928717136383f, 0.5253366231918335f, -1.6755516529083252f, 0.35308536887168884f, -0.12078773975372314f, -0.2962888479232788f, -0.08987566083669662f, -0.45044565200805664f, -1.0356383323669434f, 1.7253460884094238f, -2.3891191482543945f, -1.0028913021087646f, -0.3333589434623718f, 0.9801297783851624f, 1.2715272903442383f, -0.7523331046104431f, -0.13584411144256592f, 0.6945744156837463f, -1.9437434673309326f, 0.3301672339439392f, 1.6046744585037231f, -0.4702155888080597f, 1.209662675857544f, 0.8889638781547546f, 0.1620996594429016f, 0.12699493765830994f, 0.7177993655204773f, 0.06786083430051804f, 0.8935588598251343f, 0.11956664174795151f, 1.0848652124404907f, -0.47686854004859924f, 0.2491820603609085f, 0.730522096157074f, 0.09865526109933853f, -0.5309102535247803f, 0.08605138957500458f, -1.499106526374817f, 0.8767158389091492f, -1.1525547504425049f, -0.34564009308815f, -0.8810292482376099f, -1.5831094980239868f, -0.9634820222854614f, -1.0025994777679443f, 0.07062987983226776f, 0.11008060723543167f, 0.7735913395881653f, -0.0040825954638421535f, 0.8591998815536499f, -0.7495615482330322f, -0.38007304072380066f, -1.8807623386383057f, 0.10670913010835648f, 1.1232846975326538f, -0.15919449925422668f, -0.561027467250824f, 0.30757027864456177f, -1.5486440658569336f, -1.054305911064148f, -0.8691359758377075f, -0.06806328892707825f, -0.4993351697921753f, 0.1407848447561264f, 0.07298417389392853f, 1.068840742111206f, -1.1345824003219604f, -0.2619936764240265f, 1.4751683473587036f, 0.1493469625711441f, -0.7424628138542175f, 0.0515340156853199f, -0.4829641580581665f, -0.12765848636627197f, 0.048511698842048645f, -0.6109879612922668f, 1.034902811050415f, 1.2692794799804688f, 1.4105430841445923f, -0.2778305411338806f, -0.2591007351875305f, 1.035352349281311f, 0.4310223162174225f, -0.5226048827171326f, 1.6362823247909546f, 2.1157736778259277f, 0.22881479561328888f, 0.6379362940788269f, 0.4769587814807892f, 0.036591242998838425f, -0.7712311148643494f, 0.11032737791538239f, 1.141363263130188f, 2.2910208702087402f, 0.9097362756729126f, 0.015577469952404499f, 0.7960927486419678f, -1.0888819694519043f, -0.22314855456352234f, -0.42274436354637146f, -0.9645407795906067f, 0.26433172821998596f, -0.4537980556488037f, 1.0802478790283203f, 0.17047974467277527f, 0.5164321660995483f, 0.6420630812644958f, 0.12355764210224152f, -0.6113547086715698f, -0.07968119531869888f, 0.8360456228256226f, -0.9061261415481567f, 0.5791953802108765f, -0.9833644032478333f, 0.04046724736690521f, 0.0772174745798111f, 1.1407610177993774f, -1.1166253089904785f, 0.797391414642334f, -1.2114757299423218f, 0.845003604888916f, 0.917169451713562f, 0.6150916218757629f, -1.4388847351074219f, 0.5669911503791809f, -2.142704963684082f, -0.45466870069503784f, -0.6032977104187012f, -0.13450507819652557f, 0.18595486879348755f, 0.3553427457809448f, 1.6049801111221313f, 0.41566601395606995f, 1.1401103734970093f, 1.4051463603973389f, 0.6006536483764648f, 1.797692060470581f, -0.5229107737541199f, -0.20334042608737946f, -0.45935776829719543f, -0.15298272669315338f, 1.5941132307052612f, -0.1434730738401413f, 0.052168138325214386f, -1.3556065559387207f, 1.5627477169036865f, 1.9382938146591187f, 1.2685400247573853f, 2.026231527328491f, -1.1499669551849365f, -0.8773688077926636f, 0.07689109444618225f, 1.4310468435287476f, -0.18800176680088043f, -1.6620466709136963f, 0.719353199005127f, -0.291658490896225f, 2.5384137630462646f, 0.10889503359794617f, -0.5992977023124695f, -0.394126832485199f, -1.3852146863937378f, -1.254912257194519f, 0.1908181607723236f, -1.0126259326934814f, 0.722631573677063f, -0.34968942403793335f, -0.8847613334655762f, -0.984367311000824f, -0.9932076334953308f, 0.8806517720222473f, -0.09622633457183838f, -1.206941843032837f, -0.49620160460472107f, 0.6914839744567871f, -3.028942346572876f, 1.3682804107666016f, -0.4541301131248474f, 0.45961233973503113f, 0.36240431666374207f, 0.34856998920440674f, -0.821986973285675f, -2.4602458477020264f, 0.37095823884010315f, 0.0116643775254488f, -0.5293033719062805f, -1.6280213594436646f, -0.6806969046592712f, -0.1684180051088333f, 1.5591261386871338f, 0.43387478590011597f, 0.26209768652915955f, 1.2855982780456543f, 0.6979333758354187f, 1.5001028776168823f, -1.1095906496047974f, 0.581849217414856f, 1.3310784101486206f, 1.2475563287734985f, 1.4747735261917114f, 0.3026885390281677f, -1.3104248046875f, -0.5138301253318787f, 0.7339244484901428f, -1.0213230848312378f, -1.202392816543579f, 0.16470061242580414f, 2.572078227996826f, -1.3335460424423218f, 0.546902596950531f, 0.38991880416870117f, 0.15686734020709991f, -0.39347609877586365f, -0.30205461382865906f, 2.149627923965454f, -1.751050591468811f, -0.7567965984344482f, -1.4173346757888794f, 0.5380285382270813f, 0.0939386859536171f, 0.45474377274513245f, -0.6351379156112671f, 0.9299246072769165f, -0.4753718972206116f, 0.9432480931282043f, 0.5524624586105347f, 0.40374335646629333f, 0.6184999346733093f, 0.4535541534423828f, -1.0948821306228638f, -0.320241779088974f, -0.028895698487758636f, 0.7330871224403381f, -1.6074398756027222f, -1.227219581604004f, 1.0834808349609375f, 0.05628223717212677f, 0.20571336150169373f, 0.43112990260124207f, 0.3616485893726349f, -1.1379897594451904f, 0.2402925342321396f, -0.47906389832496643f, 1.4835309982299805f, -1.1785883903503418f, 0.5978415012359619f, 0.5928131341934204f, -0.6638352870941162f, -1.003272533416748f, -0.1671929657459259f, -1.1786675453186035f, 0.8089823126792908f, 0.1908469796180725f, 1.2084790468215942f, -2.24063777923584f, -0.5793269276618958f, -1.1207023859024048f, -1.1295276880264282f, -0.5763978362083435f, -0.26363199949264526f, 0.789819061756134f, -0.4906677007675171f, 0.371157705783844f, -1.484498143196106f, 0.2109839767217636f, -0.8684839606285095f, -1.171597957611084f, -0.9773237109184265f, -0.002813637489452958f, -1.2975432872772217f, 0.3479395806789398f, -0.7253870964050293f, -0.6621626615524292f, -0.2912011444568634f, -2.3239030838012695f, 0.918861985206604f, 1.1420683860778809f, -0.1961705982685089f, -1.5105466842651367f, 0.0809166356921196f, 1.4640601873397827f, -0.4027494490146637f, 1.6002179384231567f, -0.26083904504776f, -0.6423792839050293f, -1.7842353582382202f, -1.4978502988815308f, 1.380509853363037f, 1.877629041671753f, -1.4456785917282104f, -0.28545263409614563f, 0.15021729469299316f, 0.5569579005241394f, -1.6427204608917236f, -0.6389901041984558f, -0.5466802716255188f, -1.0836117267608643f, -1.5072098970413208f, -0.3856680691242218f, 0.9019067287445068f, -0.2348310351371765f, -1.738688588142395f, 0.28999093174934387f, -0.1533069908618927f, -0.9047854542732239f, 0.3279983699321747f, -0.5037038922309875f, -1.9761329889297485f, 0.12128926813602448f, -0.06794015318155289f, 1.1322118043899536f, -1.0514512062072754f, -0.672996997833252f, 0.549104630947113f, -0.32529518008232117f, 0.3177003860473633f, 0.1834784746170044f, 1.7332375049591064f, -0.2832452356815338f, -1.384834885597229f, -0.010661765933036804f, -0.22356857359409332f, 0.5824847221374512f, -0.6592251062393188f, 0.44558772444725037f, -2.2453958988189697f, 0.42556077241897583f, -0.6481807231903076f, -0.25188785791397095f, -1.27206289768219f, 1.1171956062316895f, 1.5053801536560059f, 0.8270988464355469f, 0.7335371971130371f, -0.7486765384674072f, 0.45417457818984985f, -0.3060213625431061f, 0.6338478326797485f, -2.674952745437622f, -0.6763451099395752f, -0.9770270586013794f, 1.5783367156982422f, -0.2843257188796997f, -1.3570187091827393f, -2.2979846000671387f, -0.8282976150512695f, -1.328344464302063f, -0.0004540873342193663f, 0.6817458271980286f, 0.9938647150993347f, -1.4253593683242798f, 0.3879549503326416f, -2.0311243534088135f, 1.2712687253952026f, 0.185286283493042f, 0.07715956121683121f, -0.584597110748291f, -0.5161778330802917f, 0.3677023947238922f, -1.3314623832702637f, 1.5407487154006958f, -0.3439099192619324f, -1.2492866516113281f, -0.8399935364723206f, 0.40630635619163513f, 0.00801965780556202f, -0.3249308466911316f, -0.6664285659790039f, 0.28344160318374634f, 0.5257611274719238f, -1.3125312328338623f, -0.3971971273422241f, -1.5499167442321777f, -0.19803878664970398f, 1.0783737897872925f, 0.5109858512878418f, 0.45442286133766174f, -0.7324386835098267f, -0.1001700609922409f, -0.9963456988334656f, 1.706158995628357f, -0.28345587849617004f, 0.24911366403102875f, -0.5420844554901123f, 0.38604170083999634f, -0.08859143406152725f, -0.7755544781684875f, -0.3989383280277252f, 0.3021530210971832f, 0.8383589386940002f, 0.36416593194007874f, -1.461991786956787f, 0.15854018926620483f, -0.5639234185218811f, 1.251044750213623f, 1.9695827960968018f, 1.165942907333374f, -1.0710505247116089f, -1.4054443836212158f, 2.9302103519439697f, -0.23575513064861298f, 0.08357129991054535f, -0.09462077915668488f, -1.8668535947799683f, -0.34727850556373596f, -0.35426023602485657f, 1.4054851531982422f, 1.4522364139556885f, -0.08649589866399765f, 0.04004557058215141f, -0.10994749516248703f, -0.6241140961647034f, -0.24008060991764069f, -0.39534568786621094f, 0.007234930992126465f, 0.03980718180537224f, -1.2252846956253052f, -0.5507245063781738f, -1.171595811843872f, 1.0313161611557007f, -0.20263050496578217f, 1.3937491178512573f, 1.0930505990982056f, 1.4651625156402588f, 0.2634923756122589f, -0.4779359698295593f, -1.4054735898971558f, 0.7812330722808838f, 0.22484587132930756f, 0.7241055965423584f, -0.6603827476501465f, -0.6927292346954346f, -0.7928091883659363f, -0.4532349705696106f, 0.5121238231658936f, -0.3005983531475067f, -0.44323164224624634f, -0.3902406692504883f, -1.0369137525558472f, -0.19362880289554596f, 0.49349725246429443f, -0.43515950441360474f, -0.6069482564926147f, -1.473711609840393f, -2.3762154579162598f, 1.134980320930481f, -0.9291712641716003f, -0.62174391746521f, 0.47012677788734436f, 1.0591752529144287f, 0.3450745940208435f, -0.04975222051143646f, 1.2706385850906372f, 0.950104296207428f, 0.9791541695594788f, -0.0728548914194107f, 1.3175126314163208f, 0.5881410837173462f, 0.9367974996566772f, 0.9128003120422363f, 0.4015538990497589f, 0.1372641772031784f, 1.041069507598877f, 0.7456277012825012f, 0.0962943509221077f, -1.7508755922317505f, -1.171151041984558f, 1.6682186126708984f, -0.6773730516433716f, -2.147958517074585f, 0.4557051658630371f, 0.061060722917318344f, 0.4539925456047058f, -1.1904879808425903f, 0.2646864056587219f, -0.8139405250549316f, -2.294616460800171f, -0.28324073553085327f, 1.7303272485733032f, -0.46407395601272583f, 1.7161751985549927f, 0.8312178254127502f, 0.2559703290462494f, 0.7086811661720276f, 0.6478610038757324f, -0.06174112856388092f, 0.2679769694805145f, -0.6163668632507324f, -0.4823808968067169f, 0.3113662600517273f, -1.5075656175613403f, -0.32805025577545166f, 0.9543997049331665f, 0.8449776768684387f, -0.8963518738746643f, 0.354059636592865f, -0.15964199602603912f, -0.20036070048809052f, 0.4515315592288971f, 0.987306535243988f, -0.9755827188491821f, 0.2618931531906128f, -2.5518243312835693f, 1.015307068824768f, 2.9806666374206543f, -0.5756134390830994f, -0.42303135991096497f, -1.6033560037612915f, -0.3883368968963623f, -1.6651710271835327f, -1.7486891746520996f, 0.5733898282051086f, -0.17000924050807953f, -1.2242252826690674f, -0.7774103879928589f, -0.662408173084259f, 1.8391462564468384f, 0.4153856635093689f, 0.07132068276405334f, 0.595319926738739f, -0.8092124462127686f, 0.3273322880268097f, -0.4520135819911957f, 0.15657921135425568f, 2.195984125137329f, 0.9013379216194153f, 0.23789522051811218f, -1.9253568649291992f, -0.8211831450462341f, -0.16658450663089752f, -2.5792624950408936f, 1.1836506128311157f, -0.0034865080378949642f, -0.5498691201210022f, 0.3581060767173767f, 0.7652292251586914f, -0.7733491659164429f, 0.6297365427017212f, -0.6815900802612305f, 1.240025281906128f, 0.8412518501281738f, 0.29200538992881775f, -1.805619716644287f, 1.3460670709609985f, 0.02373058721423149f, -0.6370474696159363f, 1.3327691555023193f, -1.63896906375885f, 0.4103865623474121f, 0.25587576627731323f, 1.880447506904602f, 1.9977002143859863f, -1.038170576095581f, 1.879461407661438f, -0.9597489833831787f, -1.338854193687439f, -1.4890133142471313f, 0.9830695986747742f, -0.41451194882392883f, 1.213753581047058f, 0.5297332406044006f, -0.8235613107681274f, 0.3100223243236542f, 0.7781904339790344f, -0.7196370363235474f, 0.8093208074569702f, -0.7462528347969055f, 0.3914867341518402f, 1.1797000169754028f, -2.474013328552246f, 0.43964120745658875f, -0.10444001853466034f, 0.5850116610527039f, -1.6503483057022095f, -1.7112841606140137f, -0.08656994998455048f, -0.3778044879436493f, -0.3667309880256653f, 0.08071848005056381f, -0.865123450756073f, 0.6451484560966492f, -0.5293470621109009f, -1.4819915294647217f, 1.995280146598816f, -0.6921202540397644f, 1.3029942512512207f, -0.1476615071296692f, 0.15694653987884521f, -0.6405912041664124f, -0.8447388410568237f, 0.3645228147506714f, -0.7561046481132507f, 1.417391061782837f, 0.5760944485664368f, 0.13722558319568634f, -0.41890090703964233f, -1.0182898044586182f, 0.448284387588501f, 1.421317458152771f, -0.6346070170402527f, -1.0847539901733398f, -0.5952850580215454f, -0.36421963572502136f, -1.5131834745407104f, -1.9063634872436523f, -1.7777600288391113f, -0.5803453326225281f, -0.6087163090705872f, 0.4910713732242584f, -0.54014652967453f, 1.2756917476654053f, 0.9132878184318542f, -1.101527452468872f, -0.8013262152671814f, -1.9939244985580444f, 1.0690128803253174f, 0.7788016200065613f, 0.35621726512908936f, -0.47195300459861755f, 0.9912806153297424f, -0.21498170495033264f, 1.6797508001327515f, -0.7104259729385376f, 0.06523545831441879f, 0.9159647822380066f, 2.1436846256256104f, 0.4262869358062744f, 0.2694464921951294f, 0.9167301654815674f, 0.6317405700683594f, 0.07572164386510849f, 1.160980463027954f, -0.6461572051048279f, 0.03240058198571205f, -0.7329845428466797f, 0.5573175549507141f, 0.1672649085521698f, -0.30756810307502747f, -0.27301856875419617f, 0.2073671668767929f, 0.1735081523656845f, -0.8468038439750671f, 0.1224641352891922f, 0.020374706014990807f, 1.1085376739501953f, 1.3970056772232056f, -0.5763441920280457f, -1.7790907621383667f, -1.4155789613723755f, -0.11962825059890747f, 1.434804081916809f, -0.6779595613479614f, 0.08594394475221634f, 0.17818263173103333f, 0.8717845678329468f, -0.4843977093696594f, 0.1896892935037613f, 1.0048247575759888f, 0.5574866533279419f, 0.20942160487174988f, -2.0887160301208496f, -1.1161184310913086f, -0.5786607265472412f, 0.8086466789245605f, 0.8664202690124512f, 1.088623046875f, 0.01803939789533615f, 0.968483567237854f, -0.3800842761993408f, 0.3059806525707245f, 0.21581949293613434f, -0.4931473731994629f, -0.15840917825698853f, 0.09949837625026703f, 0.30463987588882446f, 0.9820282459259033f, 2.1544744968414307f, 0.14304663240909576f, -1.460897445678711f, -0.1060129776597023f, -0.37611088156700134f, -0.7917583584785461f, -0.08468178659677505f, 0.8363486528396606f, -1.3825470209121704f, 0.6469754576683044f, 0.5219618082046509f, -0.6058419346809387f, -1.969196081161499f, -0.9246644377708435f, -0.508442759513855f, -1.044813632965088f, 0.353022038936615f, 0.8511560559272766f, -2.0522587299346924f, -0.02700735814869404f, -0.7895724773406982f, -2.2979352474212646f, 1.6485515832901f, -2.076538562774658f, 0.5667335987091064f, 1.3000932931900024f, 0.34126174449920654f, -0.384304404258728f, -0.2445336878299713f, 0.6715037226676941f, -0.3841925263404846f, -0.8124385476112366f, 0.4127030372619629f, 1.1925286054611206f, -1.131065011024475f, 0.3341250419616699f, -0.5277276635169983f, 0.24021567404270172f, -0.18079474568367004f, 0.9263569712638855f, -2.2063262462615967f, 0.6430982351303101f, 1.010899543762207f, -1.5004045963287354f, -0.6776713728904724f, 0.4190317988395691f, -0.1862286925315857f, 0.49416670203208923f, -0.9246350526809692f, 0.5439687371253967f, 1.2584720849990845f, -0.27466103434562683f, -1.1287552118301392f, 0.3993513286113739f, -1.8577489852905273f, 1.882336139678955f, 0.28583550453186035f, -1.3191593885421753f, -0.04691770300269127f, 1.424126148223877f, 0.47822806239128113f, -0.22128069400787354f, -0.16418692469596863f, 1.2590702772140503f, -1.5019725561141968f, -1.2077754735946655f, -0.5713672637939453f, -0.29712751507759094f, -0.08878864347934723f, -1.2249038219451904f, 0.9701720476150513f, 0.02799459546804428f, 2.49943208694458f, -0.5693896412849426f, 0.2992815375328064f, 0.4264316260814667f, -1.8905819654464722f, 0.5416959524154663f, 0.5037822723388672f, 0.06191536784172058f, 0.7051315307617188f, 1.1914323568344116f, 0.7841975092887878f, -0.5143458247184753f, 0.10756650567054749f, 0.03601418063044548f, -1.586909532546997f, -0.36298084259033203f, 0.05457804724574089f, -1.8995989561080933f, -0.4206298589706421f, 1.725341796875f, 0.2990115284919739f, 0.2708130478858948f, 1.202155351638794f, -1.3849509954452515f, 0.37148424983024597f, 1.2834670543670654f, 0.40953299403190613f, 1.5861903429031372f, 2.2006518840789795f, 1.1788570880889893f, 0.04892049357295036f, 0.45504313707351685f, -1.1838287115097046f, 0.2520817816257477f, -1.0377240180969238f, -1.1767886877059937f, 1.5097410678863525f, 0.6526481509208679f, 0.5476107001304626f, -1.4937485456466675f, -0.714340090751648f, -1.4507311582565308f, 0.3059408664703369f, 1.3935821056365967f, -2.0411956310272217f, -1.7664213180541992f, 1.1327776908874512f, 0.7594925165176392f, 0.8204308748245239f, 0.7248584032058716f, 0.6390634179115295f, -0.5571966767311096f, -0.40140989422798157f, -0.22562873363494873f, -0.5097048878669739f, 0.9061921834945679f, -0.15940190851688385f, -0.4242385923862457f, -0.23000852763652802f, -0.5473349094390869f, 0.3962896168231964f, -1.0022584199905396f, -1.9527696371078491f, -0.9625334739685059f, 0.7991358637809753f, -0.010661646723747253f, 0.023252639919519424f, 0.6258861422538757f, 0.3616181015968323f, -0.2700897455215454f, 1.6419692039489746f, -0.7898518443107605f, -0.32855224609375f, 0.9272356629371643f, 0.23767738044261932f, 2.069274663925171f, -0.6138983368873596f, 1.5534266233444214f, 0.562792956829071f, 0.9539335370063782f, 0.34314191341400146f, 0.32635602355003357f, -1.0357731580734253f, -0.021849583834409714f, 0.4444722533226013f, 1.132546067237854f, -1.3696054220199585f, -0.42242521047592163f, 0.6405972242355347f, 0.026323553174734116f, -1.8021960258483887f, -0.17883431911468506f, 0.6420789957046509f, 0.17955924570560455f, -0.2522568106651306f, -1.4123497009277344f, -0.7108830809593201f, -0.39913156628608704f, 0.6917201280593872f, -1.742424488067627f, 0.37175261974334717f, 1.2827181816101074f, -0.1333233267068863f, 2.0086865425109863f, -0.2341606765985489f, 0.7056452035903931f, 0.3473411798477173f, 0.809699535369873f, -3.0800278186798096f, -1.1150798797607422f, 1.4471967220306396f, 0.812851071357727f, 0.4065439999103546f, -0.545810341835022f, -0.6789452433586121f, -0.7672881484031677f, 0.17289385199546814f, 0.40247759222984314f, 0.6123214364051819f, -0.6715307831764221f, 0.18855510652065277f, 1.0615094900131226f, -0.2978079319000244f, -0.4801141619682312f, -1.3616151809692383f, -0.9356100559234619f, 0.978323757648468f, 1.4345933198928833f, 0.956223726272583f, -0.9492759704589844f, -0.9247362613677979f, 1.4304224252700806f, -1.2141680717468262f, 1.5573612451553345f, -2.772749185562134f, -2.07552170753479f, 0.019730882719159126f, -1.6115124225616455f, 0.2347842901945114f, -0.6375554203987122f, -0.2960043251514435f, -0.41874298453330994f, -1.3194098472595215f, 0.05774686485528946f, 0.30748531222343445f, -0.5826377272605896f, -0.31068262457847595f, 0.39234694838523865f, 1.9441745281219482f, 2.0378758907318115f, 0.4360438287258148f, -0.09107904881238937f, -0.24155908823013306f, 0.45299890637397766f, -0.10979556292295456f, 1.179660439491272f, 1.1283513307571411f, -0.8159404993057251f, -0.07127540558576584f, -0.8951465487480164f, 0.31810882687568665f, 0.6084815859794617f, 0.19665859639644623f, -1.3970006704330444f, 1.1369043588638306f, -0.11328133195638657f, 0.12461294233798981f, -0.5991875529289246f, -1.8511996269226074f, 1.1412599086761475f, -1.323111653327942f, -1.642134666442871f, -0.7886322736740112f, 0.016923261806368828f, 0.26853591203689575f, -0.08590231090784073f, -1.8297702074050903f, 1.8586663007736206f, 0.6306189298629761f, -2.0020906925201416f, -0.9767837524414062f, -1.0065470933914185f, 1.5179888010025024f, -0.25139299035072327f, 0.2684777081012726f, 0.2379687875509262f, -1.7342220544815063f, 0.890319287776947f, -0.038704462349414825f, -1.1842435598373413f, -0.6990516185760498f, -0.6046251654624939f, 1.8508250713348389f, -2.0493991374969482f, -1.7228929996490479f, 0.23755493760108948f, 0.9717959761619568f, -0.3798253536224365f, 0.685245931148529f, -1.1771284341812134f, -0.02216072380542755f, -0.1269814372062683f, -0.26579511165618896f, 0.22640825808048248f, -0.760986864566803f, 0.4831203520298004f, 1.5869791507720947f, -0.6875012516975403f, -1.2886475324630737f, 1.2204482555389404f, -0.5210925936698914f, 2.4901089668273926f, 0.15757296979427338f, -0.6784350275993347f, 1.6848376989364624f, -0.762600839138031f, 0.4223840832710266f, 0.05342012271285057f, -1.407547950744629f, 0.7069153189659119f, 0.5444626212120056f, -0.19080407917499542f, 0.14213266968727112f, 0.20559696853160858f, 1.2673413753509521f, 1.375124216079712f, 0.060717396438121796f, 0.31265851855278015f, -0.1293685883283615f, -1.0525683164596558f, -0.5546149015426636f, 1.0222612619400024f, -0.8595789074897766f, 1.0922929048538208f, -0.7460381984710693f, 0.660965621471405f, 0.454408198595047f, -1.4719254970550537f, 0.376058965921402f, -0.7192203998565674f, -0.9688137173652649f, 0.4969656765460968f, 0.42114102840423584f, -0.3883015811443329f, 0.9459248185157776f, 0.22322431206703186f, -0.3906600773334503f, 0.5856817960739136f, 0.2909909188747406f, -1.3024108409881592f, 1.4306825399398804f, 0.4534458816051483f, 0.9106372594833374f, -0.341061532497406f, -0.7916259765625f, -1.1452873945236206f, -0.4609397351741791f, -0.5050092339515686f, 1.0576668977737427f, 0.8928799033164978f, 0.7486283779144287f, 1.0225255489349365f, 1.0325298309326172f, 0.772061824798584f, 0.6529440879821777f, 1.3521305322647095f, 1.9143685102462769f, -1.3935517072677612f, 0.9990993142127991f, 1.297232985496521f, 2.667762517929077f, 0.888896107673645f, 0.8407386541366577f, 1.3103235960006714f, 1.3254046440124512f, 0.7249560952186584f, 1.3489890098571777f, 0.7757742404937744f, 0.16486017405986786f, -1.9583953619003296f, 2.0979738235473633f, -1.1749324798583984f, -1.5510687828063965f, 0.6880179643630981f, -0.27131080627441406f, -2.00603985786438f, -0.14031292498111725f, 0.24766799807548523f, -0.19394119083881378f, -0.08344011753797531f, 1.1331552267074585f, 0.4335167407989502f, 1.659290075302124f, 0.645967423915863f, -0.5915296077728271f, 1.6242163181304932f, -0.9242795705795288f, 1.5664156675338745f, -1.430296540260315f, 0.24254998564720154f, -0.9536606073379517f, 0.2823708951473236f, 0.9416194558143616f, 0.9761032462120056f, -0.14037710428237915f, 0.21578916907310486f, 0.42492198944091797f, 0.9744104146957397f, -0.6820825934410095f, 0.7403110861778259f, -0.014602077193558216f, -1.9035120010375977f, -0.459785133600235f, -0.22833675146102905f, 0.2052573710680008f, 0.060564182698726654f, -0.6736571192741394f, -0.811122477054596f, -0.011842876672744751f}; - -// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g) -// = gemm_Bpack_dram + (nt*(K/4)+g)*64. -static float gemm_Bpack_dram[64*64] __attribute__((section(".data"))) = -{-0.5196930766105652f, 1.8524175882339478f, 1.8365377187728882f, 2.074131727218628f, -0.7373097538948059f, -0.7686780691146851f, -0.05119974538683891f, 1.5985578298568726f, 0.2122737318277359f, 1.1059595346450806f, 1.311963438987732f, 0.424176424741745f, -0.4922901690006256f, 1.6562608480453491f, 0.4111401438713074f, -0.2428770512342453f, 2.081841230392456f, 1.0677173137664795f, -1.4276772737503052f, -0.33180344104766846f, 1.7054011821746826f, 0.6059234738349915f, 1.1122153997421265f, -0.5635794997215271f, -1.364527702331543f, 0.17917323112487793f, 0.5652397871017456f, 0.3271985352039337f, 0.01357425469905138f, 2.691838502883911f, 1.2729166746139526f, -0.6343013048171997f, -0.10098669677972794f, -0.43878018856048584f, -1.9775099754333496f, -0.03591495007276535f, -0.787344753742218f, -0.11653880029916763f, 1.899356484413147f, 0.45238569378852844f, -0.5850009918212891f, 0.17411360144615173f, -0.2031504362821579f, -1.271364688873291f, 0.8729038834571838f, 0.14136412739753723f, -1.4058030843734741f, 0.4083256125450134f, -1.3799183368682861f, -0.21034009754657745f, 0.8919094204902649f, 0.18319325149059296f, 0.5042280554771423f, -1.007485270500183f, -1.3393090963363647f, -0.4502287805080414f, -0.16011619567871094f, -1.608611822128296f, 0.1806594282388687f, 1.864193320274353f, -1.201043963432312f, -0.07190026342868805f, 0.14386875927448273f, -0.1494988203048706f, 0.6289454698562622f, -0.0685209259390831f, -1.2115036249160767f, -0.5285655856132507f, -1.5568547248840332f, -1.2001843452453613f, 0.2736855447292328f, -0.6129935383796692f, 0.9846591353416443f, 1.6213051080703735f, 0.5066256523132324f, -0.126494362950325f, 0.03450252488255501f, 1.3147404193878174f, -0.6877241134643555f, -0.10271778702735901f, -1.401864767074585f, -0.134507417678833f, -0.5799044966697693f, 1.0302584171295166f, 0.16683164238929749f, 0.13005995750427246f, 0.3381371796131134f, 1.025931715965271f, -1.1549469232559204f, 0.21058911085128784f, 0.0752936601638794f, 0.4396262466907501f, 0.029413584619760513f, 0.18214739859104156f, 0.03471093624830246f, -1.8955014944076538f, 0.9624823927879333f, 0.04533285275101662f, -2.03769850730896f, -1.0922635793685913f, -0.17383840680122375f, -1.065432071685791f, 0.926749587059021f, -0.16234397888183594f, 0.36853712797164917f, -0.2603956162929535f, 0.12695534527301788f, -1.7026076316833496f, 0.6422180533409119f, -0.6823133230209351f, -1.6458345651626587f, 0.15697401762008667f, -1.2265124320983887f, 0.7861263155937195f, -2.095715284347534f, 0.4538240432739258f, -0.4390734136104584f, 0.4836112856864929f, 0.9263717532157898f, 0.06930986046791077f, 1.4818024635314941f, -0.9132123589515686f, -1.606928825378418f, -0.5966755747795105f, 0.7912061810493469f, 1.1138908863067627f, 1.4326399564743042f, 0.34999290108680725f, 0.7664128541946411f, 0.4324735403060913f, -1.018404245376587f, 0.012393372133374214f, 0.6695152521133423f, 1.2033629417419434f, 0.8298293352127075f, -0.9191604256629944f, 0.958231508731842f, 0.2724153697490692f, 0.9915789365768433f, 0.3825709819793701f, -0.5029784440994263f, 1.5228251218795776f, -2.502131938934326f, -0.6252055764198303f, -0.5682758688926697f, -1.4246463775634766f, 0.4983586370944977f, -1.3945032358169556f, -2.804393768310547f, 0.4811320900917053f, 0.3296896517276764f, 1.4936070442199707f, 1.691656231880188f, 0.1595890074968338f, 1.024785041809082f, 0.12853768467903137f, -0.46378275752067566f, 1.3842717409133911f, -0.6843701601028442f, -0.3655944764614105f, -0.34340038895606995f, 0.9877837896347046f, 1.3732529878616333f, -0.32952919602394104f, -0.540728747844696f, 0.5175154209136963f, -1.1346107721328735f, 0.4711856544017792f, 1.920310378074646f, 0.4751400351524353f, 0.5673654675483704f, 0.2462792843580246f, 2.888267755508423f, -0.5556330680847168f, 0.017246929928660393f, -0.3958292305469513f, -0.3625917434692383f, 1.179726004600525f, 0.8268131613731384f, 1.457729697227478f, -1.029435634613037f, -1.2957319021224976f, 0.2842133045196533f, -2.20194673538208f, 0.5491127967834473f, -1.2930694818496704f, 0.4510805904865265f, -0.3953653573989868f, 1.7161552906036377f, 0.15392881631851196f, -1.4651801586151123f, -0.5170696377754211f, -0.789655864238739f, -1.313944935798645f, 1.211775779724121f, 0.9332025647163391f, -0.2605237364768982f, 1.5151708126068115f, -1.521173357963562f, -1.1669979095458984f, -2.0334060192108154f, -0.08260460197925568f, 1.4479997158050537f, 0.1535591334104538f, -0.19014781713485718f, -0.9898110032081604f, 0.3790753483772278f, 1.921068549156189f, -0.13450416922569275f, 0.5259649753570557f, 1.2340812683105469f, 0.19075801968574524f, -0.19310960173606873f, 1.5767059326171875f, 0.11904352903366089f, 0.2899794280529022f, -2.0928122997283936f, -0.0447654202580452f, 0.11111800372600555f, 0.38851943612098694f, 0.5478768348693848f, -1.3495665788650513f, -0.44743525981903076f, -0.7099960446357727f, 0.9817765355110168f, 1.0983757972717285f, -0.25671127438545227f, -2.2621774673461914f, 0.16864269971847534f, -1.6143133640289307f, -0.011093960143625736f, 1.2098065614700317f, -1.4476906061172485f, 1.3612878322601318f, -1.3062708377838135f, 1.6407432556152344f, -0.33931660652160645f, 0.13347899913787842f, -0.9141297936439514f, -0.15479597449302673f, -0.699771523475647f, -1.096972107887268f, -0.55166095495224f, -0.3263216018676758f, 0.6205294132232666f, -1.4098610877990723f, -0.41539478302001953f, 3.068681478500366f, 0.3701395094394684f, -0.7284356355667114f, -0.9289583563804626f, 0.8983376026153564f, -0.48385927081108093f, 0.032833296805620193f, -0.10948953032493591f, 0.46667027473449707f, 0.6613953113555908f, -0.03297487273812294f, -0.13994985818862915f, 1.0257868766784668f, -1.9717178344726562f, 0.5998077988624573f, 1.361150860786438f, -0.173064723610878f, 0.7555463910102844f, -0.14636565744876862f, -0.09429822862148285f, 0.29657915234565735f, 0.11924134939908981f, 0.1167483851313591f, -0.5673035979270935f, -1.7532938718795776f, -1.0694472789764404f, 0.08452916890382767f, 0.33887022733688354f, -1.223060965538025f, 0.19821691513061523f, 0.38515371084213257f, -0.07315804064273834f, 0.8780375719070435f, 0.12963341176509857f, -0.6663824319839478f, 0.4921484887599945f, -0.9834228754043579f, -1.2953417301177979f, -1.5002580881118774f, 1.1558160781860352f, -0.7528656125068665f, -1.160732388496399f, -1.486362099647522f, 0.13496126234531403f, 1.354621410369873f, 1.9504122734069824f, 1.0814120769500732f, 0.7843427658081055f, 1.8615505695343018f, 0.6727956533432007f, -0.4322131276130676f, 1.8005059957504272f, 1.3338027000427246f, -0.3184575140476227f, 1.1182889938354492f, 1.8907326459884644f, 0.19988827407360077f, 0.9396787285804749f, 1.0826958417892456f, -0.6519155502319336f, 0.2863307297229767f, 0.10145679116249084f, 0.09791278839111328f, -0.8082833886146545f, 1.441809892654419f, -0.6460452675819397f, 0.04811352491378784f, 1.382570505142212f, -0.6665758490562439f, 0.7876792550086975f, -0.15957334637641907f, -0.10303135216236115f, 0.27407389879226685f, -0.5971476435661316f, -1.0348221063613892f, 0.42101818323135376f, -0.548947811126709f, -1.2540193796157837f, -1.2696188688278198f, 0.6492337584495544f, 0.2340298593044281f, -0.1553155928850174f, -0.4344737231731415f, -1.0962450504302979f, -1.122981309890747f, -1.865479826927185f, -2.1207547187805176f, -0.6719433665275574f, -0.7559810280799866f, 0.1943359375f, 1.0423648357391357f, -1.1257092952728271f, -0.04083564877510071f, 0.7159355282783508f, 0.3418067395687103f, -0.6778577566146851f, 0.17783714830875397f, -0.16209769248962402f, -0.026765741407871246f, -0.33800795674324036f, 0.6160297393798828f, -1.5755101442337036f, -1.1856271028518677f, 0.3281179666519165f, -0.03522142022848129f, -0.5061670541763306f, 0.8149546980857849f, 0.6836134195327759f, -1.2236212491989136f, 0.9091914892196655f, -0.6015433669090271f, 0.8525826930999756f, -0.6866192817687988f, 1.1305443048477173f, -0.3458130359649658f, -0.673958420753479f, -0.16243989765644073f, 1.8315014839172363f, -0.7265217900276184f, -0.44100794196128845f, -1.1563644409179688f, -1.1872515678405762f, -0.83109050989151f, -1.1144185066223145f, 1.2617039680480957f, 0.6814237833023071f, 2.0168917179107666f, 0.481290727853775f, 0.313506156206131f, -0.5229234099388123f, -0.26092126965522766f, -2.14499831199646f, 1.0247118473052979f, 2.1447153091430664f, -0.34384623169898987f, 0.9131002426147461f, -2.4607415199279785f, 1.2518025636672974f, -0.35988521575927734f, -0.43994462490081787f, 1.9879144430160522f, -0.692777693271637f, 0.4603208601474762f, 0.6009213924407959f, 0.8063201308250427f, 0.979621946811676f, 0.9170622229576111f, -0.11666277050971985f, 0.2106197327375412f, 0.5228466987609863f, -1.7686034440994263f, -0.8117372393608093f, -0.877277135848999f, 0.9701831936836243f, -1.7244060039520264f, 1.61734139919281f, -0.9298601746559143f, 0.41942062973976135f, 2.0680532455444336f, -1.2342560291290283f, 2.678502321243286f, 0.3004297614097595f, -1.303833246231079f, 0.7512189745903015f, 0.3729371726512909f, 0.17139992117881775f, 0.5995815396308899f, -0.19726276397705078f, -0.11891679465770721f, -1.0388710498809814f, -0.1431705802679062f, 1.1298640966415405f, 0.587632417678833f, 0.020525068044662476f, 0.12478796392679214f, 0.2550857663154602f, -0.4017896056175232f, -1.7806396484375f, -0.08095061779022217f, -1.759868860244751f, 0.820906937122345f, -0.15008316934108734f, -1.8686583042144775f, -0.6503500938415527f, 0.6934462189674377f, 1.4084579944610596f, -2.2690858840942383f, 0.39256006479263306f, -0.4266415536403656f, 0.36491602659225464f, 0.807181715965271f, -1.5602378845214844f, -0.055706802755594254f, -0.712527871131897f, 0.2629498243331909f, 1.1325268745422363f, 0.8828094601631165f, 0.9763681888580322f, 0.35255447030067444f, -0.5103265047073364f, 0.048219867050647736f, -0.8320447206497192f, 0.9133552312850952f, 0.35530006885528564f, 1.5981814861297607f, 0.6848441362380981f, -0.238090381026268f, -0.8003812432289124f, 0.22974559664726257f, -0.19079573452472687f, -0.364820271730423f, -0.658343493938446f, -1.2190016508102417f, -1.0552600622177124f, -0.9370644688606262f, 0.07260560989379883f, 0.24500113725662231f, 1.690338134765625f, 1.8711999654769897f, 1.2940467596054077f, 0.49941954016685486f, -0.5331865549087524f, -0.26645612716674805f, 0.3932536244392395f, 0.45422324538230896f, 0.4487703740596771f, -0.43473342061042786f, 1.8583357334136963f, 0.35827821493148804f, 0.1845020353794098f, 1.472187876701355f, 0.9056950807571411f, -2.0051839351654053f, -1.232080101966858f, 1.510047435760498f, -0.4955300986766815f, -1.451595664024353f, 1.1304244995117188f, -0.23605898022651672f, -1.4099090099334717f, 0.033967722207307816f, -0.27385643124580383f, 0.052069131284952164f, 0.6194980144500732f, 1.2993793487548828f, 0.37411054968833923f, 0.29060807824134827f, -0.14929574728012085f, 0.16823826730251312f, 1.7945916652679443f, -0.8427640199661255f, -0.776123583316803f, 0.7065202593803406f, 0.020995743572711945f, -0.7148111462593079f, -0.21817894279956818f, 0.6545652151107788f, -1.3288897275924683f, -0.5552407503128052f, 0.869994044303894f, 0.17624177038669586f, 0.10479936003684998f, 1.1782784461975098f, -0.8539279699325562f, 0.18701297044754028f, -0.3077951669692993f, 1.3164128065109253f, 0.7719241976737976f, 2.0046799182891846f, -0.05154263228178024f, -0.09019286930561066f, 0.9585400819778442f, -1.2275644540786743f, 0.35432279109954834f, -0.2604968845844269f, 0.6541568636894226f, -0.43890517950057983f, -0.05930287018418312f, -1.202606201171875f, -1.2769453525543213f, -0.3968345820903778f, 0.5857738256454468f, -0.3441990613937378f, 0.6814432144165039f, 0.772460401058197f, 0.007647526450455189f, -0.25153985619544983f, 0.84389328956604f, -0.2621386647224426f, -0.42434096336364746f, -0.6080525517463684f, 0.011438353918492794f, 0.0011970907216891646f, 0.672935426235199f, -0.6712173819541931f, -0.2299986481666565f, 0.13960936665534973f, 0.6251678466796875f, -1.8366930484771729f, -0.45854899287223816f, 0.49610286951065063f, -0.8524967432022095f, -0.19241304695606232f, 0.47062209248542786f, -0.7075293660163879f, 0.01930052787065506f, 0.7421151995658875f, -2.1299924850463867f, -1.8519654273986816f, -0.789726972579956f, -1.3833003044128418f, -1.1265941858291626f, 0.4917159378528595f, 0.744454026222229f, -0.761397659778595f, 1.7994297742843628f, 0.6710255146026611f, 1.5392361879348755f, -0.8462389707565308f, -1.374268651008606f, -0.29935309290885925f, -1.3537677526474f, -0.9374998211860657f, 0.966241180896759f, 2.2905983924865723f, 0.4570314586162567f, 1.168905258178711f, -0.22257985174655914f, 1.2293546199798584f, -0.4278397262096405f, -0.527760922908783f, -1.798736572265625f, -0.07677789032459259f, 0.781619668006897f, 0.13910506665706635f, -1.7339553833007812f, 0.7391383647918701f, 1.4115384817123413f, 0.38143935799598694f, -0.051372598856687546f, 0.867921769618988f, -1.1712745428085327f, 1.219971776008606f, -0.9404910206794739f, 0.6608859300613403f, 0.49976596236228943f, 0.9063552021980286f, -0.273310124874115f, -0.8444989919662476f, 0.06740748137235641f, -1.1771124601364136f, -2.319305896759033f, 1.0881538391113281f, 1.7532049417495728f, -1.352665662765503f, -1.717094898223877f, 0.539482057094574f, -0.4647720456123352f, 0.15511354804039001f, -0.933184802532196f, 2.0511553287506104f, 1.7320348024368286f, 0.39241987466812134f, -0.3704788386821747f, 0.8811249136924744f, 0.537391722202301f, -0.48424550890922546f, -1.1632671356201172f, 0.3275741636753082f, -1.6853573322296143f, -1.058225393295288f, 1.4543863534927368f, -0.2975543737411499f, -0.8741039633750916f, -1.0332050323486328f, -0.41522952914237976f, 1.5585501194000244f, 2.5905730724334717f, -0.14419972896575928f, -0.47694188356399536f, -0.2289033830165863f, 0.3704476058483124f, 1.024267315864563f, 0.35092905163764954f, -0.11602573096752167f, -0.5128945112228394f, 0.8613727688789368f, 2.617750883102417f, 0.8522545695304871f, 0.3442921042442322f, -0.21941426396369934f, 1.6952100992202759f, 0.7312605381011963f, -2.2600386142730713f, -1.3617544174194336f, 1.5765711069107056f, 0.1465187966823578f, 1.8045871257781982f, 1.3033205270767212f, 0.18405571579933167f, -1.9564915895462036f, 0.1487964242696762f, -0.9366428256034851f, -0.950509250164032f, -0.2993629574775696f, -0.6517760157585144f, 0.2768969237804413f, -0.006574048660695553f, -1.8791542053222656f, -1.5537861585617065f, -0.9000288248062134f, 1.180068850517273f, -1.0849788188934326f, 0.6662015318870544f, -0.7253139615058899f, 1.3528261184692383f, -0.1364641636610031f, 1.1062331199645996f, -0.2775273025035858f, -0.7301269769668579f, -0.9058868288993835f, 0.3395512104034424f, 0.954081654548645f, -0.042208630591630936f, 0.05911993607878685f, -0.3268890678882599f, 0.8507034778594971f, -0.18483993411064148f, -1.1319291591644287f, -0.03408394008874893f, 1.5095572471618652f, -0.11943084001541138f, 1.4642930030822754f, 0.8833335041999817f, 1.1377960443496704f, 0.7793198823928833f, 0.07684005051851273f, -1.4407286643981934f, 0.5401078462600708f, -0.13687263429164886f, -0.4707719087600708f, -0.23642085492610931f, 2.775019645690918f, -0.048772864043712616f, 0.9577808976173401f, -0.12049184739589691f, -0.008940046653151512f, -1.624023199081421f, -0.5534864068031311f, 0.8154556751251221f, -0.03924920782446861f, 1.350463628768921f, -0.04840381070971489f, -0.7114460468292236f, -0.7299661040306091f, -1.4113084077835083f, 0.34884995222091675f, -0.43344026803970337f, 2.270435094833374f, 2.0011656284332275f, -2.1169581413269043f, 0.3513871729373932f, -0.9149423837661743f, -0.28648310899734497f, 1.87372624874115f, 0.07205427438020706f, 0.0007797196158207953f, -0.14003179967403412f, 0.9105306267738342f, -1.0649497509002686f, 0.45754849910736084f, 0.15436618030071259f, -0.744165301322937f, -0.51121985912323f, 0.23932333290576935f, 0.9544809460639954f, 0.16341786086559296f, -0.30999821424484253f, 0.15972714126110077f, 0.8302736878395081f, -1.421786904335022f, -0.09449474513530731f, 0.07646692544221878f, 0.15851546823978424f, -0.38410186767578125f, -1.4129188060760498f, -1.0182603597640991f, 0.4253365993499756f, 0.98757004737854f, 1.1435766220092773f, 0.042469993233680725f, -0.12866179645061493f, -0.3201303780078888f, 1.3020575046539307f, 0.05820586159825325f, -1.411534309387207f, -0.5787767171859741f, -1.1051533222198486f, 1.3300304412841797f, -0.41862916946411133f, -0.43831324577331543f, 1.0452755689620972f, 0.7565826773643494f, 1.3442822694778442f, -0.005769374314695597f, 1.3893702030181885f, -1.459441065788269f, 0.7733556032180786f, 1.281721591949463f, 1.922197699546814f, 1.5146862268447876f, -0.9400398135185242f, 1.7596426010131836f, 0.4703059196472168f, -1.1514455080032349f, -0.6767635941505432f, 0.9793415665626526f, 0.7851172089576721f, 0.9229853749275208f, -1.3558963537216187f, -0.4045674800872803f, -0.022665904834866524f, 0.04516039043664932f, -0.0492362417280674f, 0.688543975353241f, -0.6404033303260803f, -0.8518562912940979f, 0.5739071369171143f, -1.0256696939468384f, 0.5530444383621216f, 1.2585986852645874f, 0.23174597322940826f, -1.7316545248031616f, -0.300735741853714f, 0.8933703303337097f, -0.2885432839393616f, -0.7255284190177917f, 0.17296810448169708f, -1.597967267036438f, -0.8826010227203369f, -0.7285105586051941f, 1.0816147327423096f, 0.689936101436615f, 0.1465923935174942f, -0.2598642110824585f, 1.2287657260894775f, 0.4021441638469696f, 0.29235512018203735f, -1.4843645095825195f, -1.288714051246643f, 0.8229716420173645f, -1.8816033601760864f, -0.24393519759178162f, 1.1026350259780884f, 0.12240499258041382f, -0.6282588839530945f, -0.6169579029083252f, 0.8369725942611694f, -0.21276021003723145f, -1.037753939628601f, -0.5319015383720398f, -1.4479600191116333f, -0.7402247190475464f, -1.9904698133468628f, -0.8090559244155884f, -2.649178981781006f, 0.6452047228813171f, 1.6631805896759033f, -0.6097846031188965f, -0.5651611685752869f, -0.09456146508455276f, 0.2765144407749176f, 0.2960791289806366f, 1.2084519863128662f, -2.0801830291748047f, 0.8232543468475342f, -0.5484461784362793f, 0.34564146399497986f, 1.220015525817871f, -0.43524736166000366f, -1.7363234758377075f, -1.441352128982544f, 1.4536579847335815f, 1.2042756080627441f, -0.9513401389122009f, 1.7534879446029663f, -1.5643774271011353f, 0.2642883360385895f, 0.5893656611442566f, -0.6958070397377014f, -0.9822604656219482f, 0.9073857069015503f, 0.13120052218437195f, 0.3665435016155243f, 0.8943644165992737f, 0.990159809589386f, 1.0685539245605469f, 0.88179612159729f, 0.11932405829429626f, -1.6899914741516113f, -0.6570671200752258f, 1.6827791929244995f, 0.4652342200279236f, -0.9479421973228455f, -0.527360200881958f, -0.17596317827701569f, 1.684232473373413f, -0.005598508287221193f, -1.2838374376296997f, 0.16975225508213043f, 0.8452434539794922f, 0.28028908371925354f, 0.920665442943573f, 1.9825818538665771f, -1.0555790662765503f, -0.7873882055282593f, -0.31694626808166504f, -0.1027197390794754f, -1.1638461351394653f, -0.17423255741596222f, -0.9778326153755188f, 0.07384445518255234f, 0.2161293476819992f, -2.4143142700195312f, 0.6280608177185059f, 0.604034423828125f, -0.06171112880110741f, -0.27243953943252563f, 2.0528199672698975f, -0.6635909676551819f, 0.4586471915245056f, -1.6872135400772095f, 0.03186811879277229f, 2.094301462173462f, -0.38392743468284607f, 0.5313023328781128f, 2.5650343894958496f, -0.06663789600133896f, -0.08388976007699966f, 0.5069190263748169f, -0.9045271277427673f, -0.13790902495384216f, 1.3361674547195435f, 1.065782904624939f, 0.9382249712944031f, -0.5762946009635925f, 0.26595842838287354f, -0.515076756477356f, 1.8256999254226685f, -0.057207878679037094f, -0.2434438019990921f, 0.9149488806724548f, -2.118633508682251f, -1.2563103437423706f, 1.8411842584609985f, 0.8629192113876343f, -1.979547142982483f, 0.4109382629394531f, 0.054416440427303314f, 0.24019105732440948f, -0.3381706476211548f, -1.1024436950683594f, 0.2338137924671173f, -0.4720325171947479f, -1.3284554481506348f, 0.5184157490730286f, -0.48354241251945496f, 1.3702332973480225f, 0.18823130428791046f, -1.2953920364379883f, -0.13154013454914093f, 0.08959710597991943f, -0.44053706526756287f, 0.20576980710029602f, -1.0061860084533691f, 1.1124646663665771f, -0.7451666593551636f, -0.9709926247596741f, -0.005489187315106392f, 0.8581770658493042f, 0.8032210469245911f, -0.15868736803531647f, 1.6983613967895508f, -0.055956143885850906f, -0.45469725131988525f, 1.2942347526550293f, 0.1953326016664505f, -1.775277853012085f, 1.508985161781311f, -0.47618457674980164f, 1.1132714748382568f, 2.190094470977783f, 0.3530663549900055f, 0.4473947584629059f, 0.7191661596298218f, -1.9299876689910889f, 0.9975112676620483f, 1.1845413446426392f, 1.3882135152816772f, 0.1371254324913025f, -0.84199458360672f, -1.6440988779067993f, -0.06394191086292267f, -0.201401948928833f, 0.2995571792125702f, -1.3531256914138794f, 0.8332006335258484f, -0.1304839849472046f, 0.5836395621299744f, -0.38578328490257263f, -1.6579194068908691f, -0.2272949367761612f, 0.6312239170074463f, -1.346675992012024f, 0.6521760821342468f, -1.3508291244506836f, 2.1090853214263916f, -0.8325138688087463f, -0.9411794543266296f, -0.8664024472236633f, 0.3974232077598572f, 0.33468785881996155f, 0.23106975853443146f, -1.2114311456680298f, -1.2591102123260498f, -0.010216501541435719f, 0.9779312610626221f, 0.0748591348528862f, -1.8819596767425537f, 0.27785420417785645f, 0.734195351600647f, -0.37356483936309814f, -0.395193487405777f, -1.2448596954345703f, -0.4259602427482605f, -0.9261072874069214f, 0.33485373854637146f, 1.0248674154281616f, -0.6913897395133972f, -0.5325083136558533f, -2.0061066150665283f, 0.5374981760978699f, 0.4246957302093506f, -0.39820799231529236f, 0.4305093586444855f, 0.7205038070678711f, -0.2832542359828949f, 0.09373100101947784f, 1.657375454902649f, 1.129982829093933f, 0.6332632303237915f, 0.23538991808891296f, 0.08460769802331924f, 1.6273547410964966f, 1.4943406581878662f, 0.4516901969909668f, 0.681415319442749f, -2.738783597946167f, -0.04374435544013977f, -0.13210587203502655f, -0.20210233330726624f, -1.3241872787475586f, -0.1972821205854416f, 0.29198724031448364f, -1.2132630348205566f, 0.7370628118515015f, 0.9276220798492432f, 0.6291629076004028f, -0.23619088530540466f, -0.02293848805129528f, -2.780918836593628f, -0.8419364094734192f, -0.6721378564834595f, 0.13255943357944489f, -1.203613042831421f, 1.333530068397522f, -2.03551983833313f, 0.8634518384933472f, -1.4491990804672241f, 0.3142701983451843f, -1.005286455154419f, -1.3435431718826294f, 1.2677130699157715f, -1.2937111854553223f, -0.7414584755897522f, -0.3299030065536499f, 0.3301123380661011f, 0.9814369082450867f, -1.4912174940109253f, 0.5385298132896423f, 1.3361884355545044f, -0.5637743473052979f, 0.663472592830658f, 0.5684458613395691f, 0.5110347270965576f, 0.9753285646438599f, 1.9773973226547241f, -1.2740811109542847f, -0.7873809337615967f, 2.1380560398101807f, 0.3426303565502167f, 0.9504527449607849f, -1.5345426797866821f, -0.17094235122203827f, 0.9801839590072632f, -0.05862395092844963f, 2.073120594024658f, -0.3976811170578003f, -0.20158079266548157f, 1.4582887887954712f, -0.5315611362457275f, 0.8970864415168762f, -1.1392240524291992f, -1.309956669807434f, -0.0961947962641716f, -2.4108095169067383f, -0.009612545371055603f, -0.6612817049026489f, 0.8028563857078552f, -0.0704915001988411f, -1.5173195600509644f, 0.20034420490264893f, -0.24687924981117249f, 0.23627454042434692f, 0.7618011236190796f, -0.4506336748600006f, 0.3065869212150574f, 0.9810793399810791f, -0.5398741960525513f, -0.42057791352272034f, 0.4069875180721283f, -0.3191024363040924f, 0.8826714158058167f, 0.6646241545677185f, -2.1909663677215576f, 0.4844568967819214f, 0.6883501410484314f, 1.6444298028945923f, -0.09725860506296158f, -0.972576379776001f, 0.9053502678871155f, 0.3767105042934418f, 0.6639789342880249f, 0.05910481512546539f, 0.6989551782608032f, -0.16035781800746918f, -1.092200756072998f, 0.02478332258760929f, 0.4938628077507019f, -0.5733305811882019f, 0.6898083090782166f, 0.6767069697380066f, 0.5416850447654724f, -0.17832504212856293f, -0.7599223852157593f, 0.4881221354007721f, -1.4561625719070435f, 0.5495442748069763f, 0.1979023516178131f, -0.33105704188346863f, -1.3217003345489502f, -0.24350062012672424f, -0.4956028163433075f, 1.6506794691085815f, 0.7990935444831848f, 0.6958364844322205f, -0.9322998523712158f, 0.18523266911506653f, 0.039212681353092194f, -1.0234346389770508f, 0.31960463523864746f, -0.8433935046195984f, -2.1094107627868652f, -0.36244451999664307f, -0.12838764488697052f, -0.30779823660850525f, -1.713757038116455f, 0.23439815640449524f, -0.40255531668663025f, -1.402301549911499f, -0.8660640120506287f, 0.8337363600730896f, 0.1786288470029831f, 1.002273440361023f, -0.676571249961853f, 0.02751591056585312f, -1.0370279550552368f, 1.4250038862228394f, 0.3661484718322754f, 0.2926573157310486f, 0.9981994032859802f, 0.9345365762710571f, 0.08568228036165237f, -0.5905015468597412f, -0.603374183177948f, 0.6896741390228271f, 0.7225849628448486f, -0.4239010512828827f, 1.3261003494262695f, -1.067254662513733f, 0.6243192553520203f, -2.4506609439849854f, 1.3008989095687866f, -1.3570464849472046f, 0.4011816382408142f, -0.4519657790660858f, -0.5720081329345703f, 1.7311065196990967f, 0.40453165769577026f, 0.5910705924034119f, 0.5206778049468994f, 0.26302585005760193f, 1.462548851966858f, 0.806793212890625f, 2.1060609817504883f, 0.1250244528055191f, -1.0741941928863525f, -0.04460187256336212f, -0.7740861177444458f, 1.4476147890090942f, -1.8094489574432373f, 0.40884074568748474f, -0.13771961629390717f, -0.48222219944000244f, -0.9596375823020935f, 0.88399738073349f, -1.1032230854034424f, 0.4613407552242279f, -0.6266279220581055f, -0.059072766453027725f, 1.432356357574463f, -0.3214779198169708f, -0.32065340876579285f, -1.381635308265686f, -0.36659157276153564f, -1.3767681121826172f, -0.6575090289115906f, -1.0047813653945923f, 0.15763555467128754f, -2.729581594467163f, -0.12834908068180084f, -0.4433963894844055f, -1.7998695373535156f, -0.4873202443122864f, -0.360196590423584f, -0.2588382959365845f, 1.3558588027954102f, -0.6274254322052002f, 0.47283434867858887f, -0.6854686141014099f, 1.4168455600738525f, 0.8578910827636719f, 0.8829494118690491f, -0.8793548941612244f, -0.8100994825363159f, 0.6788665652275085f, 0.33716168999671936f, -0.5290454030036926f, 0.9324967265129089f, 0.6762214303016663f, 0.14378660917282104f, 2.18232798576355f, 0.037647929042577744f, 0.21775013208389282f, 2.395190954208374f, 0.4023849368095398f, 1.6874338388442993f, -2.743908405303955f, -1.9040486812591553f, 0.03124547004699707f, 0.3133164942264557f, -1.2002359628677368f, -0.22739538550376892f, -0.23840101063251495f, -0.682720422744751f, 0.46212872862815857f, -0.9165003299713135f, -1.665152907371521f, 0.7060098052024841f, 0.9033956527709961f, -0.3824882507324219f, 0.1978939324617386f, -0.032255567610263824f, -0.9255485534667969f, 1.1699923276901245f, -1.751257061958313f, 0.9016333222389221f, -1.3840047121047974f, -0.8077333569526672f, 1.2065165042877197f, 0.1894015073776245f, -0.6532776355743408f, -0.2359209805727005f, 0.6057451367378235f, -0.3558703064918518f, 1.4644410610198975f, -0.8236719965934753f, -0.3576895296573639f, -0.9380688667297363f, 1.4365062713623047f, 1.5855119228363037f, 0.39503413438796997f, -0.09881415218114853f, -1.8138774633407593f, -0.6933608055114746f, 1.3770830631256104f, 1.0369852781295776f, 0.522045910358429f, -2.008735179901123f, -0.7871567606925964f, 0.07242458313703537f, -1.9583851099014282f, -0.1682676076889038f, -0.09407275170087814f, -1.5021157264709473f, 0.9083237051963806f, -0.627318799495697f, -1.1001535654067993f, 0.8703535795211792f, 0.9143402576446533f, -0.9333758354187012f, -1.9243427515029907f, 0.5441842079162598f, 1.674156904220581f, -0.1457550972700119f, -0.663499116897583f, 1.2121490240097046f, 0.9221742749214172f, 0.5710628032684326f, -0.7049512267112732f, 2.053586959838867f, 0.9570578932762146f, -1.0222344398498535f, -0.5004768371582031f, -0.8111429214477539f, 2.0269458293914795f, -2.568801164627075f, 1.0234973430633545f, -0.5307855606079102f, 2.509202480316162f, 0.9496951103210449f, 1.4498697519302368f, 1.9099774360656738f, -0.41487377882003784f, -2.154376745223999f, -1.1843920946121216f, -0.13271500170230865f, -0.6129738092422485f, 1.0654757022857666f, 0.589937686920166f, -1.330565333366394f, 1.276455044746399f, -1.1509960889816284f, 1.4542961120605469f, 0.21978320181369781f, -0.26515746116638184f, 0.7749262452125549f, 0.12985540926456451f, 1.0439496040344238f, -1.7788974046707153f, 0.3559875190258026f, -0.05403977632522583f, 0.07127834856510162f, -0.8058504462242126f, -0.35074129700660706f, -0.17826782166957855f, -0.2625838816165924f, 0.7513576745986938f, 0.18278615176677704f, -0.9977543950080872f, 0.45807182788848877f, 0.7924372553825378f, -0.9643475413322449f, 0.46591266989707947f, 1.3539252281188965f, -1.459295630455017f, 0.37469375133514404f, 0.7460726499557495f, 0.26590028405189514f, -1.0854178667068481f, -1.5084643363952637f, 1.1576530933380127f, -0.21968594193458557f, 0.4215473532676697f, -0.32665783166885376f, 2.3412764072418213f, -0.5351880192756653f, 1.0548261404037476f, -0.4031357765197754f, 0.6454427242279053f, -0.6314883828163147f, -1.2809436321258545f, 0.009232764132320881f, 1.2330046892166138f, 0.6238497495651245f, 1.5213422775268555f, -0.4976206421852112f, -1.0449823141098022f, -0.27992430329322815f, 1.2101659774780273f, -0.9885204434394836f, 0.350207656621933f, -0.2644238770008087f, -1.0631657838821411f, -0.7474985122680664f, 0.3266002833843231f, -0.49929699301719666f, -1.4715039730072021f, 0.5023646950721741f, 0.4279913902282715f, 1.0040385723114014f, 1.708235263824463f, -0.2667044997215271f, 0.18906556069850922f, 1.0375696420669556f, 0.7793638706207275f, -0.6498270630836487f, -0.48095330595970154f, 0.13058727979660034f, 0.8354679346084595f, 1.6123450994491577f, -0.4939593970775604f, 0.30456098914146423f, 2.1306238174438477f, 0.026654772460460663f, 0.7856671810150146f, 0.8128212690353394f, 0.04477962106466293f, -1.7439521551132202f, -1.1074808835983276f, -0.6043308973312378f, 0.0144581263884902f, -0.2186170518398285f, -1.0052590370178223f, 1.2696419954299927f, 1.212203025817871f, -0.3618496060371399f, 1.1163915395736694f, 0.4140532910823822f, 0.12140734493732452f, 0.836719274520874f, -0.6638585925102234f, -2.0399112701416016f, -0.6206883192062378f, -0.9584940075874329f, -0.45995032787323f, 0.06401844322681427f, -0.4596543312072754f, -0.5983195900917053f, 0.844002366065979f, -0.6386957168579102f, -1.218558430671692f, -0.1842191219329834f, -1.7404849529266357f, -0.43988513946533203f, 0.978778600692749f, 1.003504991531372f, 1.839215874671936f, -0.7032709121704102f, -2.1381378173828125f, 0.3657694160938263f, -0.01325446367263794f, 1.1153861284255981f, -0.4465695023536682f, 1.315346598625183f, 1.038090705871582f, 2.2033896446228027f, -0.907842755317688f, 2.0937135219573975f, -0.9239965677261353f, -0.553339421749115f, 0.09259668737649918f, 1.0563279390335083f, 0.8357012271881104f, 0.39962950348854065f, -0.5224289298057556f, 1.8076858520507812f, 0.973744809627533f, 1.072774887084961f, -1.0264922380447388f, -1.1302908658981323f, -0.2908953130245209f, -1.6867245435714722f, 1.5600812435150146f, 1.8405765295028687f, -0.7730143070220947f, -0.2413354218006134f, 2.3902435302734375f, -0.5483986139297485f, 0.643738329410553f, 0.7495583891868591f, -1.2053273916244507f, 1.4264872074127197f, -1.1080999374389648f, -0.1472785621881485f, -2.2464029788970947f, 0.40276598930358887f, 0.1035822331905365f, 0.3125080168247223f, 0.19359846413135529f, -0.40304437279701233f, 0.7284680604934692f, 1.4359182119369507f, -1.437017560005188f, 0.5239248275756836f, -0.26981568336486816f, 0.017309220507740974f, 1.2778736352920532f, -0.30529654026031494f, 0.7788206934928894f, -0.08426624536514282f, 0.8460555672645569f, 1.1856203079223633f, -0.4465353190898895f, 0.8045015335083008f, 0.830817461013794f, -0.4116531312465668f, -0.19528378546237946f, 1.0318245887756348f, -0.6419956088066101f, -1.068763256072998f, -0.038084980100393295f, -0.8322976231575012f, 0.8178791999816895f, 0.14809109270572662f, -1.4732903242111206f, -0.10355143249034882f, -0.5564566254615784f, 0.3039886951446533f, 0.9035240411758423f, 0.5980544090270996f, -1.4176050424575806f, -0.0560019314289093f, -1.3687875270843506f, -0.5567623376846313f, -0.36739063262939453f, 1.5230790376663208f, 1.4708421230316162f, -0.4366856515407562f, -1.1750401258468628f, 0.24909113347530365f, 1.8527616262435913f, -1.680544137954712f, 0.659866988658905f, -0.5326191782951355f, 0.5515633225440979f, 1.6462424993515015f, 0.2259528934955597f, 0.27357611060142517f, -0.9556253552436829f, 0.43514177203178406f, -1.1005570888519287f, 0.9130839705467224f, -0.45959728956222534f, 2.1901278495788574f, -0.682377278804779f, -0.22877585887908936f, 0.7370786666870117f, -1.690212607383728f, 0.30514323711395264f, -0.0909213200211525f, -1.1586445569992065f, -0.7481539845466614f, 1.1121810674667358f, -0.4421563148498535f, 1.0719655752182007f, 0.17691805958747864f, -1.2427209615707397f, 0.10128959268331528f, 0.19372011721134186f, 0.24058400094509125f, 0.9057638049125671f, 0.11222076416015625f, -0.059362635016441345f, -2.4078524112701416f, -1.2304182052612305f, -0.026510445401072502f, 0.8861871957778931f, -0.7287083268165588f, 1.380319356918335f, 0.6684188842773438f, -1.7624090909957886f, 0.6292073726654053f, -0.7828742265701294f, -0.08022477477788925f, 0.9383800625801086f, -0.4210285246372223f, -0.49148616194725037f, -0.9306524395942688f, -1.267333745956421f, -0.6902195811271667f, 0.23923389613628387f, 1.4039400815963745f, -2.0282845497131348f, -1.0275602340698242f, 0.9761411547660828f, -0.06743653863668442f, 0.06586836278438568f, -0.7352823615074158f, 0.19450226426124573f, -0.7134578824043274f, 0.16818134486675262f, -1.1888638734817505f, -0.37875133752822876f, -0.321153849363327f, 1.3544011116027832f, -0.423284113407135f, -0.9974878430366516f, -0.9902269244194031f, -0.1406998336315155f, 0.8166614174842834f, 1.0789191722869873f, 1.7252999544143677f, 1.1660224199295044f, 0.5868375301361084f, -0.11877616494894028f, -0.34278497099876404f, -0.08240477740764618f, 0.8542949557304382f, -0.9149259924888611f, 0.5388334393501282f, -1.688096284866333f, -1.05665123462677f, -0.33759868144989014f, -0.2788163125514984f, -0.3783758282661438f, -0.2296314239501953f, -1.1831741333007812f, -1.034787893295288f, -0.8083778023719788f, -0.4777121841907501f, 0.8160690665245056f, 0.8076881766319275f, 0.5698519945144653f, -0.4374706447124481f, -0.7118549942970276f, 2.338996648788452f, -0.0368775799870491f, -0.1774366796016693f, 1.5478016138076782f, 1.0848923921585083f, -0.38529422879219055f, -0.5037281513214111f, 0.9991070628166199f, -0.715931236743927f, 0.28394055366516113f, 0.5939877033233643f, 0.31532949209213257f, 0.27048054337501526f, 0.9632524251937866f, -1.2189302444458008f, -0.06929556280374527f, 0.061022937297821045f, 0.822035551071167f, 0.38992002606391907f, -0.49170032143592834f, -1.689696192741394f, 0.05627693980932236f, -0.2405131459236145f, 0.9160073399543762f, 0.7348390221595764f, -0.6902630925178528f, -0.9186923503875732f, -0.6866548657417297f, -0.10339432954788208f, -0.32694968581199646f, 1.2450132369995117f, 0.6069359183311462f, -0.0937698483467102f, -0.004049783106893301f, 2.6462395191192627f, 1.7450222969055176f, 0.2888743579387665f, 1.1111222505569458f, 1.1171294450759888f, -0.7966887354850769f, 1.0229419469833374f, -1.0742470026016235f, -1.4498995542526245f, -0.5231860280036926f, 0.3438241481781006f, 0.20375612378120422f, -1.4713748693466187f, 0.19777953624725342f, -0.3222813904285431f, 0.5210180282592773f, 1.036444902420044f, 0.2102498859167099f, -1.1502959728240967f, -0.25832024216651917f, 0.2814403772354126f, -0.5159256458282471f, 1.522001028060913f, -0.4228188693523407f, 0.5675636529922485f, 0.47951194643974304f, -0.9628922939300537f, 0.16651970148086548f, 0.17922191321849823f, -0.6149380207061768f, 0.884855329990387f, -0.7095723152160645f, 1.5105708837509155f, 0.9143079519271851f, -1.2950124740600586f, 0.9506115317344666f, -1.3415539264678955f, 0.9584245681762695f, -1.834415316581726f, -0.5945284366607666f, -1.6709949970245361f, -1.3442386388778687f, -0.7921901941299438f, 0.1253465712070465f, -0.9972168207168579f, 0.9316890239715576f, -1.098996639251709f, -0.8827964067459106f, -0.03510960936546326f, 0.33106282353401184f, -1.535096526145935f, 1.5791680812835693f, 1.562730073928833f, 1.0339767932891846f, -0.644834041595459f, -0.7956817150115967f, 0.4747835099697113f, 0.6450149416923523f, -2.4694929122924805f, -1.2703917026519775f, 1.4028544425964355f, 0.39471593499183655f, -0.2914227247238159f, -0.04415971040725708f, 0.6573874354362488f, -1.8288997411727905f, 0.515746533870697f, -2.1813249588012695f, -0.7443293333053589f, 0.4601408839225769f, 1.2225193977355957f, -2.047572374343872f, 1.09447181224823f, 0.42030036449432373f, -0.6366481781005859f, -2.1695642471313477f, 1.1429638862609863f, 0.8955739736557007f, 1.6797635555267334f, -0.6834523677825928f, 0.8514192700386047f, -0.8422428369522095f, -1.0746608972549438f, -0.9010616540908813f, -0.6362974643707275f, -1.087014079093933f, -0.028106562793254852f, -0.1531383991241455f, 1.0988678932189941f, -0.03311381861567497f, -0.892038881778717f, -1.3073756694793701f, 0.14662222564220428f, 0.8670119643211365f, 3.2090775966644287f, -1.3412030935287476f, 0.3424357771873474f, 0.19632267951965332f, -0.1660011112689972f, -0.3333166837692261f, 1.0084987878799438f, 0.9129180312156677f, 0.5135545134544373f, -0.8764104843139648f, -1.4057948589324951f, -0.6906246542930603f, 0.3960992097854614f, -0.2924385964870453f, 0.3751489818096161f, -1.1075292825698853f, 0.37509971857070923f, -1.4790199995040894f, 0.5692160725593567f, -1.146249532699585f, -1.904903769493103f, 0.9906712174415588f, -0.9967307448387146f, -2.3042960166931152f, -1.8701173067092896f, 1.6571006774902344f, -0.5442659258842468f, 0.3686337471008301f, -0.17834435403347015f, -0.7094868421554565f, 0.24787208437919617f, 0.7445037364959717f, 1.5205289125442505f, -1.3338558673858643f, -2.0718472003936768f, -1.8923321962356567f, 2.419602632522583f, -0.24423770606517792f, -0.2663561701774597f, -0.6755290031433105f, -1.124692440032959f, 1.8602604866027832f, 0.6256695985794067f, 0.28849464654922485f, 0.8962987065315247f, 0.8840014338493347f, 0.4107360541820526f, -1.6067783832550049f, 1.4667541980743408f, -1.643362045288086f, -0.7528584599494934f, -0.8185959458351135f, 0.02291426621377468f, -0.7729488015174866f, -0.039141006767749786f, 0.30586403608322144f, -0.2691587805747986f, -0.21813122928142548f, 0.3616786003112793f, -1.017099380493164f, 0.7204318046569824f, -0.10523297637701035f, 1.8706086874008179f, 0.8992230892181396f, -0.23518913984298706f, -0.6936088800430298f, 0.38842713832855225f, -0.9864745140075684f, 0.24743980169296265f, 1.3209456205368042f, 0.5116807222366333f, 0.20717298984527588f, 0.423617959022522f, 0.8553113341331482f, -0.7523386478424072f, -0.7315530180931091f, -0.34988582134246826f, 0.14683414995670319f, 0.337516725063324f, -0.5604003667831421f, -0.3334929347038269f, 0.46565955877304077f, -0.831680178642273f, -0.8464822769165039f, -0.38647857308387756f, -0.1513155996799469f, 0.8186390399932861f, -0.25432348251342773f, -0.47535333037376404f, -0.780207633972168f, 0.778557300567627f, 0.0986286848783493f, -0.754118800163269f, 0.9054679274559021f, -1.1227487325668335f, 0.2190994918346405f, -0.09212681651115417f, 1.8318346738815308f, -0.5303875803947449f, -0.8198762536048889f, -0.8232145309448242f, 0.5506033301353455f, -1.1416276693344116f, 1.8735969066619873f, 1.3694992065429688f, -1.257182240486145f, -1.6220338344573975f, -0.5663302540779114f, 0.6658836007118225f, 0.25084203481674194f, 0.7626033425331116f, -0.9630534052848816f, -0.5952125191688538f, -0.5548334121704102f, 1.2073006629943848f, -0.726753830909729f, -0.25774624943733215f, 0.7335841059684753f, -0.6472665667533875f, 0.4566407799720764f, -0.506052553653717f, -0.07941094785928726f, 0.226389080286026f, 1.392283320426941f, -0.1649594008922577f, -0.8620851635932922f, 1.041149377822876f, -0.18962401151657104f, 2.0505990982055664f, -2.2949159145355225f, -0.8339797258377075f, 2.1567959785461426f, 1.0796829462051392f, -0.815654456615448f, 0.7114473581314087f, -0.24212278425693512f, 0.9910399317741394f, 0.551307737827301f, 0.6295360326766968f, 0.1759207546710968f, 1.6339221000671387f, 1.2599411010742188f, -0.6335592269897461f, -0.8656108379364014f, 0.2853967845439911f, -1.4132448434829712f, 1.7862261533737183f, -0.18690960109233856f, 0.45671215653419495f, 0.5091333985328674f, 0.12447594106197357f, -0.7764351963996887f, -0.9075336456298828f, 1.6591986417770386f, -0.9774589538574219f, -1.437660813331604f, -1.0792182683944702f, -1.0316091775894165f, 0.4912571609020233f, -0.9918735027313232f, 0.37739869952201843f, -0.09308940172195435f, -0.3923634886741638f, -1.9273265600204468f, -0.9188506603240967f, 1.127413034439087f, 1.4187533855438232f, -0.30123433470726013f, -1.6901737451553345f, 0.9281692504882812f, -1.123509168624878f, -1.0524919033050537f, -0.4467827379703522f, 0.523188054561615f, -0.3387533724308014f, -1.7577482461929321f, -0.6524283289909363f, -0.24884779751300812f, -1.2434697151184082f, 1.6212447881698608f, 0.15123429894447327f, 1.4580625295639038f, 0.8593719601631165f, 0.4116728901863098f, -0.3829646706581116f, 0.20710650086402893f, -0.845027506351471f, 0.2616454064846039f, -0.790607750415802f, -0.35345759987831116f, -0.4279586672782898f, 1.68895423412323f, 1.2128044366836548f, 0.9204495549201965f, 0.047616202384233475f, 0.7502005696296692f, -1.6819013357162476f, -0.036870624870061874f, -0.05854296684265137f, 0.6686491370201111f, -0.4365732967853546f, -2.3380324840545654f, 1.8583958148956299f, -1.7244455814361572f, -0.2750453054904938f, 0.4532167315483093f, 1.2953895330429077f, 0.058747299015522f, -0.8122621774673462f, 1.9493451118469238f, 0.7995834350585938f, -0.010960266925394535f, -0.13873475790023804f, -1.9486329555511475f, -0.7052865028381348f, -0.9286441802978516f, 0.5382292866706848f, -1.931458592414856f, 0.17587143182754517f, 1.2172441482543945f, 0.9002020955085754f, 1.1507850885391235f, 0.027491796761751175f, -1.4959322214126587f, -0.18092377483844757f, 0.905379056930542f, 1.720672845840454f, 0.3225809633731842f, 0.5499839782714844f, -0.6421884298324585f, 0.22866190969944f, -0.5370370149612427f, 0.5335260629653931f, -1.0596036911010742f, -2.2213568687438965f, 0.7067712545394897f, -0.21388542652130127f, -0.4895268678665161f, 0.5013054013252258f, -0.9222452044487f, 0.15941572189331055f, -1.377753734588623f, 1.15412437915802f, -1.092998743057251f, -0.8713886141777039f, 0.22347310185432434f, 0.2528358995914459f, 0.21688957512378693f, 0.4699849486351013f, 0.3267545700073242f, -0.09539508819580078f, -0.18269990384578705f, 0.3382812440395355f, -0.6900221109390259f, 0.047023314982652664f, 0.5487117171287537f, -1.9950600862503052f, -0.1581975668668747f, -0.1996922492980957f, 0.46645262837409973f, 0.5603392124176025f, 0.9167394638061523f, -1.983751893043518f, -0.015853818506002426f, 0.033756233751773834f, -1.4661213159561157f, -0.32681238651275635f, -0.5929558873176575f, 0.8652521371841431f, 0.4256573021411896f, -0.2335444986820221f, -0.7175306677818298f, -2.244840621948242f, 1.9783128499984741f, 0.3803824186325073f, 0.04183865338563919f, 0.284446656703949f, -2.161471128463745f, -0.5391823053359985f, 0.3041086494922638f, -0.3503345549106598f, 1.171104907989502f, -0.8915714621543884f, 1.7047383785247803f, -1.7130810022354126f, 1.0918960571289062f, 1.9015947580337524f, -0.2993949353694916f, 0.2431759536266327f, -0.06963914632797241f, -1.9186826944351196f, -1.1858599185943604f, 1.3959892988204956f, 0.31790241599082947f, 1.0485270023345947f, -0.8889532089233398f, -0.1494847685098648f, 0.011521455831825733f, -0.3648742437362671f, -0.14563268423080444f, 1.1415576934814453f, 2.0192317962646484f, -1.1988294124603271f, -0.08969400823116302f, 1.01350998878479f, -0.148199200630188f, -0.20676261186599731f, -0.4151966869831085f, -0.5613080859184265f, -1.5219647884368896f, -0.6867799758911133f, -1.207437515258789f, 0.27414625883102417f, 1.5931971073150635f, 0.8239222168922424f, 0.17494884133338928f, 0.37811675667762756f, -0.5250972509384155f, 1.1867752075195312f, 0.44894057512283325f, 0.6020882725715637f, 0.27788910269737244f, 0.3914108872413635f, -0.669601321220398f, -0.08302459120750427f, 0.06917848438024521f, -0.263455331325531f, 0.7421362996101379f, 0.2899683713912964f, -0.4380805194377899f, -1.3862485885620117f, -0.7712535858154297f, -0.3168620467185974f, -1.6128871440887451f, -0.9478936195373535f, -0.37378421425819397f, -0.8723973631858826f, 0.43149101734161377f, -0.7728766202926636f, -0.8030177354812622f, 0.46446937322616577f, -0.17089581489562988f, 2.706796407699585f, 0.6624157428741455f, -0.654021143913269f, 0.7278003692626953f, 0.09257330745458603f, -0.1797974407672882f, 0.7003864645957947f, -1.250577688217163f, 0.9090378284454346f, -0.15779435634613037f, -0.43905171751976013f, -0.16366083920001984f, -1.2389750480651855f, -1.0557494163513184f, 1.2461082935333252f, -0.7245869040489197f, -1.044531226158142f, 0.5626670122146606f, 0.17291614413261414f, 1.9904685020446777f, -1.181625247001648f, 0.30392396450042725f, 2.225896120071411f, 0.30583375692367554f, -0.6297805309295654f, 0.7762312889099121f, -0.5106104016304016f, -0.5062925219535828f, 0.7492969632148743f, 1.0681610107421875f, -2.636631488800049f, 0.6954406499862671f, -0.32303762435913086f, 0.5969774723052979f, -0.6247759461402893f, 1.647339940071106f, -1.7078118324279785f, 0.09625459462404251f, -0.28645211458206177f, 1.0167992115020752f, -0.6579540371894836f, -0.3999125063419342f, -1.7973577976226807f, -0.9080927968025208f, 1.1592087745666504f, -1.467360258102417f, 1.7191014289855957f, -0.3673190176486969f, 2.3575830459594727f, -0.459722101688385f, -0.058640673756599426f, -1.2418878078460693f, -0.023634955286979675f, -2.0428922176361084f, 0.19053655862808228f, 0.5756992697715759f, -0.5177616477012634f, 0.9197074770927429f, 0.9861821532249451f, 0.9352383017539978f, 0.30303436517715454f, -0.5343928933143616f, -0.2886335253715515f, 0.9647671580314636f, 1.7285969257354736f, -0.8043005466461182f, -0.28731220960617065f, -1.2584627866744995f, 0.08387812972068787f, 0.9973886609077454f, -0.7661278247833252f, -0.3684079647064209f, -0.16042770445346832f, 0.731488823890686f, -0.4989534318447113f, 0.5183284878730774f, 0.6223585605621338f, 0.017574317753314972f, 1.3758805990219116f, -0.024098770692944527f, -0.11441737413406372f, -0.2840602993965149f, -0.10638472437858582f, 0.3422453701496124f, -0.21973253786563873f, -0.09440521895885468f, -0.514839768409729f, 0.5137941241264343f, 0.3945278227329254f, 0.5612776279449463f, -0.006535662803798914f, 0.8489260077476501f, 0.4735972583293915f, 0.5418957471847534f, 0.0022529142443090677f, -0.5435486435890198f, 0.18075302243232727f, -0.7696079015731812f, -1.0346392393112183f, 0.7314628958702087f, 1.6726516485214233f, 0.35546284914016724f, 0.30347952246665955f, 0.04806268960237503f, 0.5032230019569397f, -0.025565603747963905f, -1.8534711599349976f, 1.0129450559616089f, -0.07218152284622192f, 0.03116939589381218f, -0.755984902381897f, -0.7956190705299377f, -0.6868169903755188f, 1.9091075658798218f, -0.4942598342895508f, -0.06308790296316147f, 0.9503589868545532f, 0.6692175269126892f, 0.12500713765621185f, 0.5085462331771851f, -1.0646655559539795f, -0.4647725224494934f, -0.9152674674987793f, 0.7144214510917664f, 1.472544550895691f, -1.7723859548568726f, 0.0483609139919281f, 0.20343580842018127f, -0.6890958547592163f, -0.5412123203277588f, 2.0465071201324463f, -0.5884720087051392f, -0.8352398872375488f, 0.0502433180809021f, 0.4375612735748291f, 1.3833050727844238f, -1.3894257545471191f, 0.3897654116153717f, -1.5383917093276978f, 1.8197290897369385f, -0.5204964280128479f, 1.4799479246139526f, 0.4248008131980896f, -0.30355918407440186f, 0.07046826928853989f, -1.5127924680709839f, 1.3581115007400513f, 0.32402706146240234f, -1.0680902004241943f, 0.08959023654460907f, 3.1311562061309814f, -0.8544708490371704f, -0.8357695937156677f, 1.2985942363739014f, -0.7110929489135742f, -0.9653112292289734f, -1.340206503868103f, 0.03399703651666641f, -0.4458200931549072f, 0.151767760515213f, 1.6231015920639038f, -0.429679274559021f, -0.16159531474113464f, -1.0276970863342285f, -0.9918712973594666f, 0.29410406947135925f, -0.3595946729183197f, -0.13289135694503784f, -0.09766664355993271f, -1.9935065507888794f, 0.11608057469129562f, 0.9187806844711304f, 0.12344544380903244f, 2.8273682594299316f, 0.47407886385917664f, -1.9149128198623657f, -1.5384353399276733f, -0.6529980897903442f, 0.16414928436279297f, -0.19700200855731964f, 1.038715124130249f, -0.3908529281616211f, 1.0207839012145996f, -1.2268577814102173f, -2.0579991340637207f, 0.5987722277641296f, -0.13595744967460632f, 0.24731674790382385f, -2.150991439819336f, 1.2150318622589111f, -0.33008328080177307f, -0.5156939625740051f, 0.022917015478014946f, -0.6746217608451843f, 1.309282898902893f, -1.6657313108444214f, 0.32850465178489685f, -0.8491258025169373f, 0.9747982025146484f, -0.37322181463241577f, 0.6090918779373169f, -0.7589855194091797f, -0.6492360830307007f, -1.2674946784973145f, -0.382905513048172f, 0.8315308690071106f, 1.8437397480010986f, 0.021685972809791565f, 0.28762421011924744f, 0.39912644028663635f, 2.4587013721466064f, 0.4567311406135559f, 1.106606125831604f, 0.9169553518295288f, -0.23127765953540802f, -1.498667597770691f, 0.6427391171455383f, 0.18222902715206146f, -1.0757185220718384f, 1.345549464225769f, 0.32771772146224976f, 1.3408823013305664f, 1.2158557176589966f, 0.95885169506073f, 0.5136737823486328f, 0.5763065814971924f, 0.0778578370809555f, -1.061161994934082f, 2.042306900024414f, 0.6509259343147278f, -1.0072094202041626f, 0.35781362652778625f, -1.0799492597579956f, 0.04710597172379494f, 2.201401472091675f, 0.2908516526222229f, -0.08251824229955673f, 1.5195056200027466f, -0.3584267199039459f, -0.6102548837661743f, -1.5060930252075195f, -0.7732543349266052f, 1.0847911834716797f, 1.4872429370880127f, 0.6041548848152161f, -1.1257244348526f, 0.3761575222015381f, 1.959293246269226f, -0.7958163619041443f, 0.4539487659931183f, -2.5069937705993652f, -1.361000418663025f, 1.183761477470398f, -1.1673825979232788f, -0.25077250599861145f, 0.33264845609664917f, 1.1230510473251343f, 1.8761743307113647f, 0.273639976978302f, 0.2801147401332855f, 0.7281787991523743f, 0.02403142862021923f, 2.3937904834747314f, -1.3129955530166626f, -0.48624828457832336f, 0.4012764096260071f, 1.0767757892608643f, 1.1279785633087158f, 0.02586616761982441f, -0.14947263896465302f, 0.39506977796554565f, 1.9742354154586792f, -0.4404173791408539f, 2.3815414905548096f, -0.34589144587516785f, 2.694831609725952f, -0.8544124364852905f, -0.27781322598457336f, 0.7772916555404663f, -0.43923330307006836f, 1.1729096174240112f, 1.397361159324646f, -0.41255858540534973f, -0.4339471161365509f, 0.9574882388114929f, 1.254128336906433f, -1.0384563207626343f, 0.5251074433326721f, -0.8383287787437439f, 1.7227904796600342f, -1.1731188297271729f, 0.7272883057594299f, -1.7194626331329346f, 2.383774995803833f, -1.1649107933044434f, -0.20503655076026917f, -0.5375787019729614f, 1.2967820167541504f, -1.6138769388198853f, 0.7535606026649475f, -0.38888975977897644f, 2.5921998023986816f, 0.08543383330106735f, -2.0308878421783447f, 0.6461803913116455f, 1.217452883720398f, 0.03121299482882023f, 1.082465648651123f, -1.8998514413833618f, -2.7074427604675293f, -0.7867730259895325f, -0.6195639967918396f, 1.8281400203704834f, 0.4877544343471527f, 0.1366790533065796f, -0.5910199284553528f, 2.3618080615997314f, 1.2076122760772705f, -0.5487976670265198f, 1.5311497449874878f, 0.44362834095954895f, -2.4579336643218994f, 0.6496618390083313f, 0.02449978142976761f, 0.25982725620269775f, 0.7348682284355164f, -0.006029692944139242f, -0.7884382605552673f, 1.1371253728866577f, -1.7366209030151367f, 0.9614791870117188f, -1.3104760646820068f, -1.4218922853469849f, -0.3201002776622772f, 0.011662798002362251f, 0.7528783082962036f, -0.8849666118621826f, 0.8982152342796326f, -1.6744109392166138f, -1.1447242498397827f, 0.22346419095993042f, -0.9369489550590515f, -0.17544429004192352f, 0.8354768753051758f, -1.2516489028930664f, 0.26434803009033203f, 1.2949540615081787f, -0.5020677447319031f, 0.45930108428001404f, -1.3068112134933472f, 0.9302626848220825f, 1.0709624290466309f, 1.3384746313095093f, 0.27829068899154663f, -1.328766107559204f, -0.2891634404659271f, -2.518200635910034f, -2.533468723297119f, -0.022988365963101387f, -0.5563327670097351f, 2.0752601623535156f, -0.8403553366661072f, 0.9517558813095093f, -2.1648738384246826f, 0.9794597625732422f, 0.38047143816947937f, -1.9277540445327759f, 0.9197121858596802f, -1.6067678928375244f, -1.3559657335281372f, -0.25936615467071533f, 0.0861901044845581f, -0.37659966945648193f, 0.9756653904914856f, -0.8411983847618103f, -0.9621247053146362f, -0.5808306932449341f, 1.1524169445037842f, -0.5585207939147949f, -0.13208438456058502f, -1.0929938554763794f, -2.4860122203826904f, -0.038849566131830215f, 0.8021038770675659f, -0.18547378480434418f, 0.534229576587677f, -0.49810588359832764f, 0.8963930010795593f, -1.2691730260849f, 0.042280785739421844f, 0.9680984020233154f, -1.7608284950256348f, 1.0980932712554932f, 0.8579375743865967f, -0.9562888145446777f, -0.7236151099205017f, 0.2793406844139099f, -0.8662601113319397f, 0.8459339737892151f, 0.6116387844085693f, -0.8089803457260132f, -0.20030127465724945f, -2.4035587310791016f, 1.5633418560028076f, -0.6294617652893066f, 0.8042920231819153f, 0.6676139235496521f, -0.5181459784507751f, 0.9539962410926819f, 0.7256157398223877f, -0.5767995715141296f, -0.01296310406178236f, 1.34257972240448f, 0.8595607280731201f, 0.24403470754623413f, 1.9769524335861206f, 0.5327519178390503f, -0.8788067102432251f, -0.2609003186225891f, -0.1277228742837906f, -0.38701093196868896f, -0.10875839740037918f, 0.7135027647018433f, 0.037005677819252014f, 0.7367677688598633f, -1.073968529701233f, -0.8155962824821472f, 0.3688916862010956f, 0.5232675075531006f, -1.1358050107955933f, 2.01452898979187f, -0.5778390765190125f, -0.2187574803829193f, 0.35567042231559753f, 0.7510744333267212f, 0.2171924114227295f, 0.4019394814968109f, 0.9250243306159973f, -0.24931922554969788f, -0.3473069369792938f, 0.8122217059135437f, -0.6078834533691406f, -0.8096639513969421f, -0.4163765013217926f, -0.9129456877708435f, -1.0231975317001343f, 0.15973883867263794f, 1.5596561431884766f, 0.2012278288602829f, 0.9135115742683411f, -0.7519399523735046f, 2.7958858013153076f, 0.41675102710723877f, 0.0033115341793745756f, -1.398554801940918f, 1.5899420976638794f, -0.7806249856948853f, -0.7495994567871094f, 0.2814400792121887f, -0.08736804872751236f, -1.1191902160644531f, -0.4140253961086273f, 0.2933676540851593f, -0.016588633880019188f, 0.4729222357273102f, 1.0479241609573364f, 0.32106152176856995f, 1.162279486656189f, 1.5198776721954346f, 1.134521245956421f, 0.013782661408185959f, 1.5933806896209717f, 0.11106725037097931f, 0.5933053493499756f, 0.0023419519420713186f, 0.9275448322296143f, 0.1871725618839264f, 1.0549991130828857f, -0.04648401960730553f, 1.6230204105377197f, 0.9345911145210266f, 0.6804553866386414f, 2.2367258071899414f, -0.6858885884284973f, -1.308877944946289f, -1.208367109298706f, -1.4461109638214111f, -0.6192750930786133f, -0.40478718280792236f, 0.45292869210243225f, 0.40945965051651f, 0.9656304121017456f, 0.4625106155872345f, -1.4208135604858398f, 0.2507438659667969f, -0.5702037215232849f, -0.05394743010401726f, -0.7306843400001526f, -1.0346637964248657f, 0.7631427049636841f, 0.9210829138755798f, -0.6593214273452759f, 0.24809309840202332f, 0.7311660647392273f, 2.3288936614990234f, -1.06203031539917f, -0.5385692119598389f, 0.4127616584300995f, 0.3390391767024994f, 1.7734715938568115f, 1.655669927597046f, -0.08070362359285355f, -0.303115576505661f, 0.3240552544593811f, 1.0401465892791748f, 0.5915576815605164f, -1.4803826808929443f, -0.12040789425373077f, 0.16437974572181702f, 0.7806977033615112f, 1.228264331817627f, 0.4453698992729187f, -0.3023916184902191f, -0.7796450257301331f, 1.15239417552948f, -0.4312492907047272f, 0.4804547131061554f, -0.0037577631883323193f, -0.23722761869430542f, 0.1753881573677063f, -0.3416663408279419f, -0.4271427094936371f, 1.1991220712661743f, 0.8689031600952148f, -0.6073640584945679f, 0.11616495996713638f, 1.9142365455627441f, -1.1450425386428833f, 1.60905921459198f, -0.5139697194099426f, -0.3140712380409241f, -0.8430630564689636f, -1.2868314981460571f, 0.5520419478416443f, 2.584942102432251f, 0.4215601086616516f, 2.558887481689453f, -1.2483208179473877f, -0.9826495051383972f, 0.7373914122581482f, 1.4831291437149048f, 0.6203920245170593f, 0.33499792218208313f, -0.5796453952789307f, -0.24162966012954712f, 1.4769864082336426f, -1.1677722930908203f, 0.43188926577568054f, 0.7582098245620728f, 0.26907673478126526f, -0.16138136386871338f, 1.1750508546829224f, 0.5351220369338989f, 1.225508689880371f, 0.9085782766342163f, 0.16776901483535767f, -0.009964917786419392f, -0.9104782342910767f, 0.3974350690841675f, -0.5543625354766846f, 1.281534194946289f, 0.38322561979293823f, 1.3406338691711426f, -0.43185070157051086f, -1.897344708442688f, -2.652250289916992f, -0.43282368779182434f, 0.932184100151062f, 0.4349152445793152f, -0.7768413424491882f, -0.19172848761081696f, 1.1136181354522705f, -1.214506983757019f, 2.1966660022735596f, 0.41227126121520996f, -0.7043505311012268f, 0.09847358614206314f, 1.8926005363464355f, 0.6903649568557739f, 1.4352598190307617f, 1.0864993333816528f, -0.3989373445510864f, -2.0487279891967773f, -1.1373023986816406f, 0.9046337604522705f, 0.5279921293258667f, -0.5005130171775818f, 0.0378837063908577f, -1.0129538774490356f, 0.7843693494796753f, 1.3280545473098755f, -0.5138816833496094f, -0.04832516983151436f, -0.8721303939819336f, -1.3769567012786865f, -1.8487751483917236f, 0.30561649799346924f, -0.42448726296424866f, 0.6269863247871399f, -0.6438542604446411f, 3.0598578453063965f, 0.8688741326332092f, 0.7331935167312622f, 1.7552188634872437f, -0.24250611662864685f, 1.9880743026733398f, -1.1370364427566528f, -2.2499585151672363f, -0.5810695886611938f, 0.8816671967506409f, -2.0325005054473877f, -0.35224223136901855f, 0.1006089523434639f, 1.5255436897277832f, -1.6215003728866577f, 0.5298449397087097f, -1.3718810081481934f, -0.6298099756240845f, -0.65668123960495f, 0.4244276285171509f, -0.40122607350349426f, -0.4890124797821045f, 0.9571828246116638f, -0.8436367511749268f, -0.3680597245693207f, -0.47244831919670105f, -0.6993737816810608f, 0.4523862302303314f, -1.090578317642212f, 0.9411556720733643f, -1.931666612625122f, 0.1285097301006317f, -1.0134315490722656f, 0.7479439973831177f, 0.20112790167331696f, -0.8995627760887146f, -0.2906798720359802f, 0.1047126054763794f, -0.5943686366081238f, -1.727795124053955f, 1.0557186603546143f, -0.5037193298339844f, -1.5121967792510986f, -0.14455023407936096f, -0.7709440588951111f, 1.3850905895233154f, 0.06768468767404556f, -2.205592393875122f, -0.9360390901565552f, 1.6600853204727173f, -0.551665186882019f, -0.3103943169116974f, -0.6468674540519714f, -0.22681531310081482f, -0.3377947509288788f, 0.6926270723342896f, -0.5544835925102234f, -0.10487986356019974f, -0.12251695245504379f, -0.10804381966590881f, -1.4534186124801636f, -0.5115614533424377f, 0.9907002449035645f, 0.7906493544578552f, -0.34170281887054443f, -1.8583146333694458f, 0.6896669268608093f, 0.8880993127822876f, 1.2050648927688599f, -0.5951094031333923f, -0.3993092179298401f, -0.30013880133628845f, -0.014673005789518356f, 0.6156877279281616f, -0.9164139032363892f, -0.06423450261354446f, -0.33351314067840576f, 0.5679208636283875f, 0.5063413977622986f, 1.8924111127853394f, -0.7531763911247253f, 1.0589748620986938f, -0.010384922847151756f, 0.41770482063293457f, 0.16313597559928894f, -0.26724591851234436f, 0.1958005577325821f, 0.8279790282249451f, -0.7302268147468567f, 0.4149245321750641f, 2.0801644325256348f, 0.1493149846792221f, -0.4159306585788727f, 1.2188113927841187f, 0.8605815768241882f, 0.8988009691238403f, -0.4320070743560791f, -0.011584565043449402f, 0.1905190795660019f, 0.45351630449295044f, -0.2722189724445343f, 0.4377744197845459f, 0.6674123406410217f, -0.8114345669746399f, -1.1677125692367554f, 1.4608654975891113f, -1.6388529539108276f, 0.9854339957237244f, 0.01134670153260231f, 0.3140886127948761f, 1.081308126449585f, 0.1636349856853485f, -1.8501884937286377f, -0.6727942824363708f, -1.8205945491790771f, -0.16954289376735687f, -0.974631667137146f, -0.5593003630638123f, -0.2645290195941925f, 0.2448410987854004f, 0.4654839038848877f, 1.1221051216125488f, -1.2755296230316162f, 1.968849539756775f, 1.1931818723678589f, -0.9534568786621094f, 0.6892605423927307f, -0.03748741000890732f, -1.3567429780960083f, -0.930943489074707f, -0.4104326069355011f, 1.6298463344573975f, -0.3964575529098511f, -0.8092941045761108f, -0.8157675862312317f, -0.1790880262851715f, -0.7333499789237976f, 2.0713958740234375f, 0.26833969354629517f, -0.4909171462059021f, 0.0867118388414383f, -0.19213901460170746f, -0.020139964297413826f, 1.3420833349227905f, 0.8771296143531799f, 0.9378947615623474f, 0.023886969313025475f, 1.5275574922561646f, -0.08036366105079651f, -0.9117053151130676f, -1.1944957971572876f, 0.9197462201118469f, 0.28947049379348755f, 0.5755492448806763f, -1.0247280597686768f, 0.19418835639953613f, 1.5901795625686646f, -0.7810332179069519f, 0.18075843155384064f, 0.5682910680770874f, -1.3145383596420288f, -0.037264980375766754f, -1.1747896671295166f, -0.5241218209266663f, 0.8552600145339966f, 0.7151663303375244f, 0.854972243309021f, 0.6970289349555969f, -0.7869400978088379f, -0.22646142542362213f, 0.8945121765136719f, -0.07112658768892288f, 0.19750215113162994f, -0.08805552124977112f, 0.46580448746681213f, 0.745457112789154f, -0.6745474338531494f, -0.8672813177108765f, -2.0061123371124268f, 2.204427480697632f, -0.336713045835495f, -0.07664714008569717f, 0.6008157134056091f, 0.7986276745796204f, -1.3824840784072876f, -0.3528830111026764f, 0.9141533374786377f, 0.5920729637145996f, 0.7283052206039429f, 0.5198004245758057f, 0.40039631724357605f, -0.16163653135299683f, 0.3812867999076843f, 0.2818247973918915f, -0.259854257106781f, -0.17636136710643768f, 1.2899055480957031f, -1.809929609298706f, 0.04499081149697304f, -0.5114196538925171f, 1.6922872066497803f, -0.5404288172721863f, -0.06902796775102615f, -0.900188148021698f, 0.16361606121063232f, -0.5697844624519348f, -0.17199715971946716f, 1.9494951963424683f, -1.5725066661834717f, -0.11967476457357407f, 0.6768812537193298f, -2.550161600112915f, 1.3886760473251343f, 2.071373224258423f, -0.4157007336616516f, 1.7397890090942383f, -1.5081530809402466f, -0.9471526145935059f, 0.4156656265258789f, -0.7409192323684692f, -2.0307180881500244f, 0.7459039092063904f, -0.23346112668514252f, -1.7168304920196533f, -1.5648266077041626f, 1.1608803272247314f, -1.7281652688980103f, 0.6240199208259583f, 0.19563332200050354f, 0.9567276835441589f, 0.9916862845420837f, -0.7535449862480164f, 1.3851630687713623f, 0.8076578974723816f, 0.4444909691810608f, 0.12085723131895065f, 0.794389545917511f, -0.2691969871520996f, 1.6320013999938965f, 1.4558833837509155f, 0.8513883352279663f, 1.305290937423706f, 0.8631715178489685f, -0.5379660725593567f, 0.5391438603401184f, -0.18788452446460724f, -1.6509469747543335f, 0.6196591854095459f, -1.1270289421081543f, -0.08517079800367355f, 1.0406969785690308f, -0.5241701006889343f, -0.5131880044937134f, 0.4295358955860138f, 1.8150509595870972f, -1.150861382484436f, -0.30584630370140076f, 0.8076725602149963f, 0.7956660985946655f, 0.2235465794801712f, -0.05132210999727249f, 0.7380313277244568f, 1.5020791292190552f, 0.8205595016479492f, -0.056694693863391876f, -0.17149198055267334f, 0.9732901453971863f, -0.9835246801376343f, 0.1294081211090088f, -1.1072022914886475f, -0.1295740306377411f, 1.005579948425293f, -1.3748044967651367f, 1.197999119758606f, -2.2044677734375f, -1.1458145380020142f, -1.4537255764007568f, -0.2384955883026123f, -1.6215006113052368f, -0.3137211501598358f, -0.7217759490013123f, -1.6709321737289429f, 0.3768741488456726f, -1.8265146017074585f, 0.48899734020233154f, -0.5580688714981079f, 0.37294480204582214f, -0.6878536939620972f, -1.4576401710510254f, -0.6108043789863586f, -0.6761788725852966f, -1.9144877195358276f, 0.9340372085571289f, -0.4341937303543091f, 1.4776188135147095f, 0.7693286538124084f, 2.331334352493286f, -0.6489352583885193f, 0.4903559982776642f, -0.2804304361343384f, 0.941003680229187f, -0.11338216066360474f, -0.8615462183952332f, 0.13828350603580475f, 0.013346405699849129f, 0.33505237102508545f, -0.24309676885604858f, -0.6505504250526428f, -0.33003056049346924f, 1.3748613595962524f, -1.033202886581421f, 0.05953902378678322f, -0.2076457291841507f, -0.9487320184707642f, -0.2869058847427368f, -0.902455747127533f, -1.5298161506652832f, -1.2186866998672485f, 1.1458920240402222f, 0.20151416957378387f, -0.4259151816368103f, -0.1506912112236023f, -2.106266498565674f, -0.6301612257957458f, 1.2424033880233765f, -0.025852564722299576f, -0.540851891040802f, -0.3480299413204193f, -0.7818117141723633f, 0.5524802803993225f, 0.8857057690620422f, -1.3390332460403442f, -0.40026965737342834f, 0.7468986511230469f, -0.3904872536659241f, 0.7475174069404602f, -0.4455646276473999f, 1.0798193216323853f, 2.6205930709838867f, 0.6864228844642639f, 1.5354714393615723f, -0.1950068324804306f, -1.4005751609802246f, 0.4377126097679138f, -0.05925627797842026f, -0.552335798740387f, -0.7325940728187561f, 0.5038686990737915f, -0.18185997009277344f, -0.7621927261352539f, -0.8477222323417664f, 0.6641071438789368f, 2.3913745880126953f, 0.4669826030731201f, 0.7512773275375366f, -0.3845667541027069f, -0.06630031019449234f, 0.6945987939834595f, -0.7470712065696716f, 0.2076980471611023f, -0.12689319252967834f, -0.5187065005302429f, 0.8009016513824463f, -0.7448427081108093f, 1.9110214710235596f, 0.3227183520793915f, -0.5433307886123657f, -2.0223824977874756f, 0.7815577387809753f, 1.8362489938735962f, -0.47440794110298157f, -0.38593149185180664f, 0.36727604269981384f, 0.344801127910614f, 0.5554131865501404f, 0.35217124223709106f, 0.7673863768577576f, 0.08270980417728424f, 0.3816434144973755f, -1.0023962259292603f, -0.22075708210468292f, -0.6525165438652039f, -2.485137939453125f, 0.614276111125946f, 0.9891844391822815f, -1.6560537815093994f, 0.17567120492458344f, 0.8857656121253967f, 0.022398322820663452f, -0.24084077775478363f, 1.4238216876983643f, 0.8160249590873718f, 0.2655353844165802f, 0.744394838809967f, -0.17409075796604156f, -0.9177995324134827f, -2.1325347423553467f, -0.4131704568862915f, -0.4764634966850281f, 0.5031798481941223f, 0.27043330669403076f, -0.04328599572181702f, 1.035592794418335f, 1.341705560684204f, -0.11890244483947754f, -0.07687754184007645f, 0.5947466492652893f, 0.5723721385002136f, 2.663830518722534f, 0.7388755679130554f, -0.46826601028442383f, -1.59627366065979f, -1.6671663522720337f, 0.3388274610042572f, 0.3750116229057312f, -1.3291982412338257f, 0.5636889338493347f, 0.8051766753196716f, 0.7449150681495667f, -0.1739114224910736f, 1.1078135967254639f, 0.5147720575332642f, 0.8934884667396545f, -1.5113967657089233f, -0.8514725565910339f, 0.3001462519168854f, 0.17312221229076385f, 0.08587908744812012f, 0.10766935348510742f, 1.0629868507385254f, 1.841042399406433f, -0.5686787366867065f, -1.44584059715271f, -0.2561878263950348f, 1.1728384494781494f, -0.428275465965271f, -0.2985764443874359f, 0.14890146255493164f, 1.661197304725647f, -2.622263193130493f, -0.43193310499191284f, -0.3783835768699646f, 0.23239530622959137f, -0.7321792840957642f, -0.40373992919921875f, 1.2062691450119019f, -0.6938396096229553f, -1.5164529085159302f, 2.0242106914520264f, -0.8075132966041565f, 0.09412498027086258f, 0.27298247814178467f, 0.5441054701805115f, -2.4758286476135254f, 0.21060240268707275f, 0.2536042630672455f, -0.7318422794342041f, 0.05695872753858566f, 0.15881912410259247f, 0.6510986685752869f, 0.020065616816282272f, 0.8478670120239258f, 2.0752015113830566f, -0.38090255856513977f, -0.9460003972053528f, -0.4590383470058441f, -0.6942368745803833f, 0.30009862780570984f, -1.389849305152893f, 1.2990328073501587f, -1.8040062189102173f, 1.837660551071167f, -0.7219073176383972f, -0.3056040108203888f, 0.9780712127685547f, 0.9052547812461853f, -0.901805579662323f, -0.2280527949333191f, -0.948022723197937f, -0.15365807712078094f, 1.032126545906067f, 0.17955327033996582f, -0.2721782922744751f, 0.15856090188026428f, 1.1139295101165771f, -0.5713488459587097f, -0.9870969653129578f, -1.6782877445220947f, 1.8284801244735718f, -0.4982566237449646f, 0.47082117199897766f, -0.591300904750824f, -0.41513174772262573f, 0.5321887731552124f, 1.317676305770874f, -0.7898051142692566f, -0.5088178515434265f, 0.23661386966705322f, 1.2172428369522095f, 0.2148703932762146f, 1.2481820583343506f, 2.4610743522644043f, 0.14153295755386353f, 1.837676763534546f, -0.2276618480682373f, 0.2129386067390442f, 0.8061029314994812f, 0.7021245956420898f, -0.6485925316810608f, -0.47295162081718445f, 2.2190706729888916f, -1.868390679359436f, -1.3732751607894897f, -1.013265609741211f, 1.365964651107788f, 1.1893959045410156f, -2.7800850868225098f, -1.0310310125350952f, -0.702293336391449f, 0.8099242448806763f, 0.8948581218719482f, -0.40090087056159973f, -0.08886078000068665f, -0.9508647918701172f, 0.28014296293258667f, 0.06467004120349884f, -2.057403087615967f, 1.1129963397979736f, -0.7580298185348511f, 0.0977560505270958f, 0.8891482949256897f, 0.3460270166397095f, -0.11394428461790085f, 1.4566701650619507f, -1.1309462785720825f, 1.2330803871154785f, 0.8789023160934448f, -2.017490863800049f, 0.856982946395874f, 0.04652906581759453f, -0.3498983681201935f, -0.8770002722740173f, -0.38342466950416565f, -0.05906866118311882f, 0.7605865001678467f, -0.9748004078865051f, 0.9013839960098267f, -0.43347686529159546f, -1.508413314819336f, 2.2633509635925293f, 0.88945072889328f, 0.7355136275291443f, -0.6224141120910645f, 0.2808963656425476f, 0.1529616415500641f, 0.09397149085998535f, 0.8745917677879333f, -1.226547122001648f, 1.3209213018417358f, 1.0341976881027222f, -0.49460795521736145f, -3.3865373134613037f, 2.0950191020965576f, 0.2728019058704376f, -0.030417079105973244f, 0.08612233400344849f, -1.7298319339752197f, 0.08109258860349655f, 1.2689827680587769f, -0.6729629635810852f, 0.8545035123825073f, -1.1793856620788574f, -0.9540548324584961f, -0.969673752784729f, 0.3141362965106964f, 1.2000207901000977f, -0.1831716001033783f, -0.18045181035995483f, -0.1034746766090393f, -0.10869846493005753f, 0.5782245993614197f, 1.7499538660049438f, 0.002034955658018589f, 0.36500951647758484f, -0.8946718573570251f, -0.219259113073349f, -0.47011202573776245f, -0.9500816464424133f, -0.6808398962020874f, -0.9808987379074097f, -0.7519583702087402f, -1.0324863195419312f, -1.8080289363861084f, 0.21794334053993225f, 0.6857624053955078f, 0.43733805418014526f, -1.9561761617660522f, 1.2915894985198975f, -1.6977379322052002f, -1.2025177478790283f, -1.646543025970459f, -1.3370387554168701f, 1.169712781906128f, -0.5579738020896912f, 0.38680657744407654f, -0.4910755455493927f, -0.32521501183509827f, 0.13093096017837524f, 0.624261200428009f, 0.6792762279510498f, 0.3959449231624603f, 0.66949063539505f, -0.19622361660003662f, -0.32004305720329285f, 0.20433473587036133f, -1.9621531963348389f, -0.5470468401908875f, 1.5721423625946045f, 1.0094727277755737f, 0.4029425382614136f, 1.6145533323287964f, 0.4475519359111786f, 0.5499113202095032f, 0.21999366581439972f, -0.019641423597931862f, 0.20199595391750336f, -0.305207222700119f, -0.11879883706569672f, 0.8017854690551758f, -0.3870505392551422f, 1.560797929763794f, 0.04964430257678032f, -0.33324524760246277f, 0.7588040828704834f, -1.5863133668899536f, -0.6730678081512451f, -1.2264569997787476f, 1.3401838541030884f, -1.162084698677063f, 1.4780147075653076f, 1.5603762865066528f, -0.3231433033943176f, 2.119018793106079f, 0.7877871990203857f, -1.9625552892684937f, -0.526032567024231f, 2.639968156814575f, 1.171855092048645f, 1.2722933292388916f, 0.4042589068412781f, -1.5217982530593872f, -0.5133728981018066f, -1.3806458711624146f, -0.5332193970680237f, 1.0821458101272583f, 0.6407410502433777f, -1.729612946510315f, -2.14782977104187f, 0.4973359704017639f, -1.268904685974121f, -0.24232769012451172f, -1.0745846033096313f, -0.6819858551025391f, 1.0070197582244873f, -0.8279359936714172f, 2.587069511413574f, -0.7386665940284729f, 0.3296529948711395f, 2.8114237785339355f, -0.879487931728363f, -0.33060166239738464f, 0.6457027792930603f, 0.17840361595153809f, -1.1289931535720825f, -0.3602311313152313f, -2.383305549621582f, 1.3506853580474854f, -0.4065081775188446f, -0.30150434374809265f, -1.5348396301269531f, 1.6021355390548706f, 0.6893559694290161f, 0.033526014536619186f, -0.2977294325828552f, -1.1700760126113892f, 1.5237687826156616f, 1.2984881401062012f, 0.6588833928108215f, -0.8222662210464478f, -0.14632517099380493f, 0.7788861393928528f, 0.6403340101242065f, 0.8075235486030579f, 0.5731524229049683f, 1.3903805017471313f, -0.23513318598270416f, -0.5386273264884949f, -0.8155969977378845f, -0.1509992778301239f, -0.5020706653594971f, 0.8036778569221497f, -1.1333380937576294f, -0.7435185313224792f, -0.7013434171676636f, -0.42178988456726074f, -1.5332636833190918f, -1.9342981576919556f, 0.3510490953922272f, -0.6404718160629272f, 0.019135747104883194f, -0.7149955034255981f, -0.7025302648544312f, 0.7700265645980835f, -0.7571792006492615f, -1.0642281770706177f, 1.9611531496047974f, -0.4390997588634491f, 0.8920031785964966f, 1.8720558881759644f, 0.24501417577266693f, 0.7017510533332825f, 0.9099079370498657f, -0.7323962450027466f, -0.6663155555725098f, -0.3277064561843872f, -1.439155101776123f, -1.2797472476959229f, -0.19622953236103058f, 0.19903564453125f, -0.48808231949806213f, -0.6568363904953003f, -0.4720822274684906f, 0.35971710085868835f, 0.5307507514953613f, 1.0079243183135986f, 1.6095112562179565f, 1.6016820669174194f, 0.020138248801231384f, 0.573236346244812f, 0.37203314900398254f, 0.22609540820121765f, -0.754463255405426f, -0.3379971385002136f, 0.41659975051879883f, 0.22890278697013855f, -0.5443984270095825f, 2.050001859664917f, 0.06459449976682663f, 1.448185920715332f, -0.20686893165111542f, 1.5211719274520874f, -1.2341058254241943f, -0.7871248126029968f, -0.7268313765525818f, -0.26047298312187195f, -0.1815493106842041f, 0.40858983993530273f, 0.21912771463394165f, -0.45708730816841125f, -0.651996910572052f, -0.32107922434806824f, 0.20424777269363403f, -0.8860291838645935f, 1.0496968030929565f, 0.5568186640739441f, 0.13751773536205292f, -0.15331187844276428f, 1.2169883251190186f, 1.7785981893539429f, -0.709164023399353f, -0.9840890765190125f, -0.17089858651161194f, 0.2695762813091278f, 0.3193606436252594f, -0.6905565857887268f, 0.3500501811504364f, -0.32124459743499756f, -0.25051894783973694f, 2.1708250045776367f, -1.229917049407959f, 0.3085583448410034f, -0.4612124264240265f, -1.0031081438064575f, -3.3694987297058105f, 0.7832650542259216f, -0.34354400634765625f, 0.34807446599006653f, -1.7184555530548096f, -0.8929248452186584f, -0.7456709742546082f, 0.1508672833442688f, -0.7863383889198303f, -0.350765198469162f, -0.04678618907928467f, -0.6908267736434937f, -0.38857290148735046f, 0.04591083899140358f, -0.3794901967048645f, 1.5770217180252075f, -0.042833227664232254f, 1.353956699371338f, -1.124684453010559f, -0.17863909900188446f, 0.20412735641002655f, 0.9273927807807922f, -0.32438376545906067f, 0.6851024031639099f, 1.0722298622131348f, -1.000272512435913f, -0.8011578917503357f, -1.122711181640625f, -0.7824243903160095f, -0.5122944712638855f, -0.8242338299751282f, -1.0234198570251465f, -1.3928337097167969f, -0.4097520112991333f, 0.8509801030158997f, 0.8881950378417969f, 0.5827952027320862f, 0.785405158996582f, 1.5136889219284058f, 1.251064419746399f, -1.9372137784957886f, -1.2051048278808594f, 1.1982215642929077f, -1.376950740814209f, 1.0670830011367798f, 0.507300615310669f, 0.4764442443847656f, 0.9100666046142578f, -1.501094937324524f, -0.11335238814353943f, -1.2001420259475708f, -0.7933986186981201f, -2.7309763431549072f, -0.014749551191926003f, -0.04001638665795326f, -0.11494863778352737f, 1.458335041999817f, 0.4836253225803375f, 1.7024855613708496f, -0.9683796763420105f, 1.086437702178955f, 1.8661012649536133f, 1.4962280988693237f, 1.022226333618164f, 1.5485637187957764f, -0.08855976164340973f, -0.88787841796875f, -0.6835475564002991f, -0.015076662413775921f, -0.030384592711925507f, -0.8306879997253418f, -1.7835168838500977f, -1.239241361618042f, 0.9374039173126221f, -0.12884187698364258f, 0.9388713836669922f, 0.2919924259185791f, 1.7478625774383545f, -0.8145643472671509f, 0.9247024655342102f, -1.5529402494430542f, 0.13943159580230713f, 1.0945836305618286f, 1.0349615812301636f, 0.03250877186655998f, 0.9967503547668457f, 0.6327362060546875f, 0.7831454873085022f, 0.5098334550857544f, 1.5633010864257812f, -0.46283742785453796f, 0.1497601568698883f, 0.6901775598526001f, -0.38219600915908813f, 0.4878943860530853f, 0.04518410563468933f, 1.9969353675842285f, -0.6955253481864929f, 0.5120656490325928f, -0.9795308709144592f, 2.3521711826324463f, 0.41813287138938904f, -0.1308819055557251f, -0.4394487142562866f, 1.6976182460784912f, 0.813541829586029f, -0.6087417602539062f, 0.9832689762115479f, 2.428079843521118f, 1.1440294981002808f, -0.101145900785923f, 0.09917446225881577f, 0.3814586102962494f, 1.6723263263702393f, -0.4240947961807251f, 1.8199645280838013f, -0.39397287368774414f, -0.4608776867389679f, 0.51866215467453f, 0.7784189581871033f, -1.5908970832824707f, 1.9877948760986328f, 0.02497357502579689f, -0.43002068996429443f, 1.0171878337860107f, -0.548503041267395f, 1.6083859205245972f, 0.5863268971443176f, 0.15758942067623138f, -0.9828112125396729f, 0.18478159606456757f, 1.4150960445404053f, 0.4230518639087677f, 0.5434484481811523f, -1.2116377353668213f, 0.1757367104291916f, 0.25499454140663147f, 0.8070029616355896f, -0.11587464064359665f, 0.45107510685920715f, -0.3600947856903076f, -1.0252388715744019f, 0.37429946660995483f, -0.3791492283344269f, 2.650177001953125f, -1.6031533479690552f, -0.47765374183654785f, -0.5845633149147034f, -0.21724168956279755f, -0.5731971859931946f, 0.37175968289375305f, -0.4610559642314911f, 2.174776315689087f, 0.021914366632699966f, 0.1974239945411682f, -0.7603867650032043f, -0.693874180316925f, -1.0808472633361816f, -0.06485684216022491f, -0.010432514362037182f, 1.2084461450576782f, -0.9010456800460815f, -1.3052830696105957f, -0.12543383240699768f, 1.2808754444122314f, 0.6577393412590027f, 0.46201997995376587f, -0.5072460770606995f, -0.9180205464363098f, 0.0884491354227066f, -1.0534428358078003f, 0.2707360088825226f, -0.25277644395828247f, 0.016658902168273926f, -0.2014150619506836f, 0.05029870569705963f, 1.4853851795196533f, 0.8810616731643677f, 0.9759232997894287f, 0.807918131351471f, -1.3484669923782349f, 1.1456364393234253f, 0.31596988439559937f, -0.4552096426486969f, -0.18135374784469604f, -0.3125359117984772f, 2.018014669418335f, -0.44383805990219116f, -0.7552081942558289f, -1.7850168943405151f, -0.07548937201499939f, -0.5207358598709106f, -1.048349142074585f, 0.8424208760261536f, -0.9976255297660828f, 0.7335740327835083f, 1.2276335954666138f, -0.7577790021896362f, 1.622873306274414f, 1.292340874671936f, -0.26265472173690796f, -0.8315040469169617f, 1.0420149564743042f, 1.154740810394287f, 0.31027355790138245f, -0.6142987608909607f, 1.667606234550476f, 0.014784318394958973f, -1.3251757621765137f, -0.8346104025840759f, -0.2093493789434433f, 0.2113834023475647f, -0.15933676064014435f, 1.438138484954834f, -0.8965448141098022f, 0.4655975103378296f, -0.7802034020423889f, -0.7512221336364746f, -0.20733071863651276f, 0.41877850890159607f, 1.7955039739608765f, -0.6994057297706604f, -0.4367755055427551f, 0.34429505467414856f, -0.5315167307853699f, -1.7878962755203247f, 0.5759621858596802f, -0.37107211351394653f, -0.02148612029850483f, -0.3246121108531952f, 0.7541995644569397f, -1.7302361726760864f, -0.6404775977134705f, -0.024382900446653366f, -0.7490391731262207f, -2.1543679237365723f, -0.15692853927612305f, 0.09731115400791168f, 0.2538682818412781f, 1.5049229860305786f, 1.313053011894226f, -0.7395356297492981f, -1.813647985458374f, -0.2881675958633423f, -0.4023030996322632f, 0.7287482619285583f, -0.49943476915359497f, 0.7028453350067139f, 0.34216001629829407f, -1.3413481712341309f, 0.41704463958740234f, -0.8712550401687622f, -0.16223306953907013f, -0.9481205940246582f, -0.6534963846206665f, -0.3585118353366852f, -0.817564845085144f, 0.3736099898815155f, 0.9122175574302673f, 0.6781302690505981f, 0.055852312594652176f, -1.3394618034362793f, -0.30016884207725525f, 0.15999306738376617f, -0.6769115328788757f, 0.7504494786262512f, 0.216787651181221f, 0.23502039909362793f, 1.8141969442367554f, -1.5329066514968872f, -0.3652367889881134f, -1.5126714706420898f, 0.7930607199668884f, -1.0631688833236694f, 0.9984860420227051f, 1.037604570388794f, -0.04009038954973221f, -1.5908479690551758f, 0.30711954832077026f, 1.1175891160964966f, -1.345697283744812f, 1.0572247505187988f, 1.3808448314666748f, -0.366502046585083f, -0.08894097805023193f, -0.662988007068634f, -0.291555255651474f, 0.10108345001935959f, 1.0123193264007568f, -0.815822958946228f, -0.18586060404777527f, -0.7641056776046753f, -0.3749435842037201f, -0.8628830313682556f, -0.8098255395889282f, -0.21431559324264526f, -1.1151552200317383f, 0.14144384860992432f, -0.17559614777565002f, -0.7265163660049438f, -0.5217191576957703f, 0.37568187713623047f, 0.5935606956481934f, 0.20674307644367218f, 1.4827877283096313f, 0.9587778449058533f, -0.3417870104312897f, -1.0058722496032715f, -1.0774322748184204f, -2.2288825511932373f, 0.8113047480583191f, 0.13623955845832825f, -0.30091696977615356f, -0.2654571533203125f, -1.8403681516647339f, 0.1881687343120575f, -1.4522165060043335f, 1.4667081832885742f, 0.9349684119224548f, 0.25753054022789f, 0.9158172607421875f, -0.5319824814796448f, 0.6224501132965088f, 0.12310407310724258f, -0.17781615257263184f, 2.9330546855926514f, 0.4627670347690582f, -0.6264359354972839f, -1.3771589994430542f, -1.6635388135910034f, -0.2668311595916748f, -1.2189315557479858f, 0.14258776605129242f, -0.31570670008659363f, 1.423683524131775f, 1.0125263929367065f, -0.7147458791732788f, 0.3446122407913208f, 0.8802022337913513f, -0.7164220213890076f, -1.3230516910552979f, 1.0481867790222168f, -1.7053356170654297f, 0.600459098815918f, 1.8295810222625732f, -0.340763121843338f, 0.34682390093803406f, -0.1821630895137787f, -0.12917080521583557f, 0.7775542140007019f, 0.3030397295951843f, -0.8394461274147034f, 0.4900873601436615f, 0.9002130031585693f, -1.4860044717788696f, 0.9374179840087891f, -0.5453174114227295f, 0.485347181558609f, 0.6994611024856567f, -0.28342336416244507f, -0.11592065542936325f, 0.22056539356708527f, -0.8155755400657654f, 1.1422392129898071f, 0.16099725663661957f, 1.1248847246170044f, 0.09503551572561264f, 0.6680463552474976f, 0.22494570910930634f, -0.8516902327537537f, 0.4524131119251251f, -0.7356429100036621f, 1.461809754371643f, 0.056686531752347946f, -0.6240271329879761f, -0.3087240159511566f, 0.18612052500247955f, 0.026730623096227646f, -0.8790956139564514f, 1.3807705640792847f, -0.6990379691123962f, -1.2263665199279785f, -1.9690755605697632f, 2.3512303829193115f, 0.35396817326545715f, -0.10606183111667633f, -2.408507823944092f, -1.2640177011489868f, -0.4252816140651703f, 0.24079595506191254f, -0.24080689251422882f, -0.6830735802650452f, 0.28334593772888184f, -0.7549530863761902f, 0.685922384262085f, -0.28879889845848083f, 0.16147172451019287f, -0.18717071413993835f, 0.7061312794685364f, 0.12637053430080414f, -0.2261367291212082f, -0.40762385725975037f, -0.6097135543823242f, -1.347667932510376f, 0.8310710787773132f, 0.23480524122714996f, 0.23941050469875336f, -0.15233780443668365f, 1.35030198097229f, 0.3163374066352844f, -0.686985969543457f, 0.31536293029785156f, -1.2173928022384644f, -0.23235148191452026f, -0.11819366365671158f, -0.7065277099609375f, 0.8570377826690674f, -0.6065717935562134f, -2.5028295516967773f, -0.5181497931480408f, -0.3955882489681244f, -0.007207863032817841f, -1.8654048442840576f, -1.2592604160308838f, 0.44495517015457153f, 1.5878770351409912f, -1.009257197380066f, 2.5517168045043945f, 0.27151748538017273f, -0.45277684926986694f, 0.7783017754554749f, 0.5624421238899231f, -1.7339200973510742f, 0.1151348203420639f, 0.5969645380973816f, 1.4009592533111572f, 1.26650869846344f, 0.5665708780288696f, 1.4182316064834595f, 0.8785027265548706f, 1.713185429573059f, 0.5062318444252014f, 0.9417456984519958f, 1.1770522594451904f, -0.8855880498886108f, -0.897435188293457f, -0.4500139355659485f, -1.2818336486816406f, -0.09055130183696747f, 1.568543553352356f, 0.6288840770721436f, -0.13983508944511414f, -0.8341673016548157f, -1.0642942190170288f, -0.4966285824775696f, -0.8965734243392944f, 0.2338687777519226f, 0.13322870433330536f, -1.087693452835083f, -0.6877674460411072f, -0.034093379974365234f, -1.2961610555648804f, 1.018326997756958f, 2.094313621520996f, 0.22312399744987488f, 0.04033441096544266f, -0.11042066663503647f, 1.209588646888733f, 0.8461918830871582f, 1.0998334884643555f, -2.8670525550842285f, 0.43270260095596313f, 2.3624584674835205f, -0.9460180401802063f, 0.40504685044288635f, 0.21921862661838531f, -1.154799461364746f, 1.519834041595459f, -1.8380980491638184f, 1.218536376953125f, -0.1873958259820938f, 1.0788236856460571f, 0.14058129489421844f, -0.9915911555290222f, -1.5487273931503296f, -1.3351367712020874f, -1.668448805809021f, -1.84798002243042f, 0.02599930204451084f, 0.7676464319229126f, -1.9874215126037598f, 0.9982309937477112f, 0.003224122803658247f, 0.07818755507469177f, -1.0705227851867676f, -1.0210168361663818f, 1.7321865558624268f, 0.5420076847076416f, 0.3053724765777588f, 1.1364459991455078f, 0.6844477653503418f, 0.2691081464290619f, 0.8210340738296509f, 0.27511492371559143f, -0.38933637738227844f, -0.20226091146469116f, -0.19409379363059998f, -0.07234596461057663f, -0.7745715975761414f, 0.2601335644721985f, 1.2001798152923584f, -1.078745722770691f, 1.2962092161178589f, -0.18015773594379425f, -1.9301296472549438f, 0.28462305665016174f, 0.7962566614151001f, -0.26019519567489624f, 0.30697566270828247f, -0.7991265654563904f, 0.6080788373947144f, -0.8028495907783508f, -0.46709370613098145f, 0.5347719192504883f, 1.8560413122177124f, -0.11687781661748886f, 0.6693155169487f, -0.06095214933156967f, -0.1428632140159607f, -1.3090907335281372f, 0.03258459270000458f, 1.4419121742248535f, -0.6330651044845581f, -0.03063541278243065f, 1.5843851566314697f, -0.467591792345047f, -0.03230065852403641f, 0.7200000286102295f, -0.988913357257843f, -1.0607333183288574f, 0.7062268257141113f, -0.2287754863500595f, -0.030369358137249947f, 0.7900131344795227f, -0.22298280894756317f, 1.508110761642456f, 0.599722146987915f, -0.33899757266044617f, 0.1548950970172882f, -0.2794382572174072f, 0.8708680272102356f, 0.7626234889030457f, 0.6961768865585327f, 0.44085901975631714f, -0.4885202944278717f, 0.7180350422859192f, 1.8486416339874268f, -2.431525230407715f, -0.4701055586338043f, 0.4380730092525482f, 0.7702670693397522f, -0.14850954711437225f, 0.24941392242908478f, 0.539443850517273f, 0.058982912451028824f, 0.28753769397735596f, -0.6253249645233154f, 1.5168472528457642f, -0.610085129737854f, -1.8375158309936523f, 1.8140860795974731f, 0.9514070153236389f, -1.2033350467681885f, -0.5142884254455566f, -0.8533352017402649f, -0.5986783504486084f, -0.2986593544483185f, -0.7190134525299072f, -1.4356969594955444f, 0.6340908408164978f, -0.5407924652099609f, 1.3264530897140503f, 0.29634353518486023f, 0.17156629264354706f, 0.8127211332321167f, -0.636906087398529f, -1.1077786684036255f, 1.5005162954330444f, 0.5292592644691467f, -1.5582870244979858f, -0.22035345435142517f, -0.8878109455108643f, -0.23165416717529297f, -0.48117324709892273f, -0.8784040212631226f, 0.039074741303920746f, 0.03500647097826004f, -1.3603416681289673f, -0.3714776337146759f, -0.040006835013628006f, 0.5846149921417236f, 0.5806360840797424f, 0.7450182437896729f, 1.4870532751083374f, -0.051454752683639526f, 0.2527291476726532f, 0.7950483560562134f, 1.2641831636428833f, -0.0403965562582016f, -0.19124414026737213f, -0.4378063976764679f, -0.009368259459733963f, 0.25679612159729004f, -0.003075298387557268f, -1.1646342277526855f, -0.46010419726371765f, 0.18153296411037445f, 0.18499712646007538f, -0.23162956535816193f, 2.05483078956604f, 0.4368106722831726f, 1.1093194484710693f, 1.684390902519226f, -0.11415659636259079f, 0.37000924348831177f, 0.2512214779853821f, 1.1637510061264038f, -0.876596987247467f, -0.6737048625946045f, -0.652714192867279f, -0.753913164138794f, -0.10021805018186569f, -2.3608896732330322f, 2.2464849948883057f, -1.1070212125778198f, 1.026766061782837f, 0.2865336239337921f, -0.10950178653001785f, 2.380542039871216f, -0.41917189955711365f, 0.6831162571907043f, 1.7866971492767334f, -0.022699790075421333f, 0.7491136193275452f, -1.1757766008377075f, 0.2672518789768219f, -0.22947388887405396f, 0.5409282445907593f, 0.6939815282821655f, 1.8563321828842163f}; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/data/layer.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/data/layer.h deleted file mode 100644 index 54e4009f2..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/data/layer.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2024 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include - -/** - * @brief VME GEMM layer configuration. - * - * Describes a C = A * B problem with: - * A : M × K (stored row-major; kernel receives pre-transposed At : K × M) - * B : K × N (stored row-major) - * C : M × N (fp32 accumulator, tew=32) - * - * TM / TN are the micro-tile sizes used by the 2×2 blocked VME kernel. - * Typical values: TM = TN = 16. - */ -typedef struct { - uint32_t M; /**< Rows of A / rows of C */ - uint32_t N; /**< Columns of B / columns of C */ - uint32_t K; /**< Inner (reduction) dimension */ - uint32_t TM; /**< Micro-tile height (M direction) */ - uint32_t TN; /**< Micro-tile width (N direction) */ -} vme_gemm_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/kernel/gemm.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/kernel/gemm.c deleted file mode 100644 index c073c9ec6..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/kernel/gemm.c +++ /dev/null @@ -1,607 +0,0 @@ - -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include "gemm.h" - -enum { - GEMM_TSS_MT0_ROW0 = 0u << 27, - GEMM_TSS_MT4_ROW0 = 4u << 27, - GEMM_TSS_MT8_ROW0 = 8u << 27, - GEMM_TSS_MT12_ROW0 = 12u << 27, -}; - -__attribute__((noinline, aligned(64))) void gemm_fp32( - float *C, const float *Apack, const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi) -{ - const int ti = (int)ti_lo; - const int row_blocks = (int)gemm_l.M / 64; - const int col_blocks = (int)gemm_l.N / 32; - const int num_blocks = row_blocks * col_blocks; - const int col_tiles = (int)gemm_l.N / 16; // TILE_DIM - const int tile_stride = 16 * (int)gemm_l.K; - - // The M64 two-core partition assigns one row-tile pair to each core. - (void)ti_hi; - - if (ti_lo != 0) - asm volatile("nop" ::: "memory"); - - uintptr_t a0p = (uintptr_t)(Apack + ti * tile_stride); - uintptr_t a1p = (uintptr_t)(Apack + (ti + 1) * tile_stride); - uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); - uintptr_t tss0 = GEMM_TSS_MT0_ROW0; - uintptr_t tss4 = GEMM_TSS_MT4_ROW0; - uintptr_t tss8 = GEMM_TSS_MT8_ROW0; - uintptr_t tss12 = GEMM_TSS_MT12_ROW0; - uintptr_t p00 = (uintptr_t)(C + ti * col_tiles * 16 * 16); - uintptr_t p01 = p00 + 16 * 16 * sizeof(float); - uintptr_t p10 = (uintptr_t)(C + (ti + 1) * col_tiles * 16 * 16); - uintptr_t p11 = p10 + 16 * 16 * sizeof(float); - const uintptr_t tile_dim = 16; - const uintptr_t TK = 8; - uintptr_t block_counter = (uintptr_t)num_blocks; - uintptr_t loop_counter; - uintptr_t vtle_counter; - - asm volatile( - - // First Block - // K-Group 0 - // vle A0 - // vle B0 - "msetmtypei 1, 2\n" - "msettn x0, %[tile_dim]\n" - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" - "vle32.v v16, (%[b0p])\n" - "msettm x0, %[tile_dim]\n" - "msettk x0, %[tk]\n" - "li %[vtle_loop], 16\n" - ".Lvtle_first_mt0_%=:\n" - "vtle32 %[tss0], (%[p00])\n" - "addi %[p00], %[p00], 64\n" - "addi %[tss0], %[tss0], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_first_mt0_%=\n" - "addi %[p00], %[p00], -1024\n" - "addi %[tss0], %[tss0], -16\n" - - // vle B1 - // vtfmm mt0, A0, B0 - "vle32.v v24, (%[b1p])\n" - "vtfmm.tvv mt0, v0, v16\n" - "addi %[a0p], %[a0p], 512\n" - "vtfmm.tvv mt0, v1, v17\n" - "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt0, v2, v18\n" - "addi %[b1p], %[b1p], 512\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "li %[vtle_loop], 16\n" - ".Lvtle_first_mt4_%=:\n" - "vtle32 %[tss4], (%[p01])\n" - "addi %[p01], %[p01], 64\n" - "addi %[tss4], %[tss4], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_first_mt4_%=\n" - "addi %[p01], %[p01], -1024\n" - "addi %[tss4], %[tss4], -16\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtfmm.tvv mt0, v7, v23\n" - - // vle A1 - // vtfmm mt4, A0, B1 - "vle32.v v8, (%[a1p])\n" - "vtfmm.tvv mt4, v0, v24\n" - "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtfmm.tvv mt4, v2, v26\n" - "li %[vtle_loop], 16\n" - ".Lvtle_first_mt8_%=:\n" - "vtle32 %[tss8], (%[p10])\n" - "addi %[p10], %[p10], 64\n" - "addi %[tss8], %[tss8], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_first_mt8_%=\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "addi %[p10], %[p10], -1024\n" - "addi %[tss8], %[tss8], -16\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtfmm.tvv mt4, v7, v31\n" - - // preload vle A0 - // vtfmm mt8, A1, B0 - "vle32.v v0, (%[a0p])\n" - "vtfmm.tvv mt8, v8, v16\n" - "addi %[a0p], %[a0p], 512\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "li %[vtle_loop], 16\n" - ".Lvtle_first_mt12_%=:\n" - "vtle32 %[tss12], (%[p11])\n" - "addi %[p11], %[p11], 64\n" - "addi %[tss12], %[tss12], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_first_mt12_%=\n" - "addi %[p11], %[p11], -1024\n" - "addi %[tss12], %[tss12], -16\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - - // preload vle B0 - // vtfmm mt12, A1, B1 - "vle32.v v16, (%[b0p])\n" - "vtfmm.tvv mt12, v8, v24\n" - "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 1 to N-3 - // vle B1 - // vtfmm mt0, A0, B0 - "9:\n" - "li %[loop], 5\n" - "1:\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "vle32.v v24, (%[b1p])\n" - "vtfmm.tvv mt0, v2, v18\n" - "addi %[b1p], %[b1p], 512\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtfmm.tvv mt0, v7, v23\n" - - // vle A1 - // vtfmm mt4, A0, B1 - "vle32.v v8, (%[a1p])\n" - "vtfmm.tvv mt4, v0, v24\n" - "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtfmm.tvv mt4, v7, v31\n" - - // preload vle A0 - // vtfmm mt8, A1, B0 - "vle32.v v0, (%[a0p])\n" - "vtfmm.tvv mt8, v8, v16\n" - "addi %[a0p], %[a0p], 512\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - - // preload vle B0 - // vtfmm mt12, A1, B1 - "addi %[loop], %[loop], -1\n" - "vle32.v v16, (%[b0p])\n" - "vtfmm.tvv mt12, v8, v24\n" - "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group K-2 & K-Group K-1 - // vle B0' - // vle A0' - // vtfmm mt0, A0, B0 - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "vtfmm.tvv mt0, v2, v18\n" - "vle32.v v24, (%[b0p])\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v8, (%[a0p])\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtfmm.tvv mt0, v7, v23\n" - "addi %[a0p], %[a0p], 512\n" - "addi %[b0p], %[b0p], 512\n" - - // vle B1 - // vtfmm mt0, A0', B0' - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v16, (%[b1p])\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - "addi %[b1p], %[b1p], 512\n" - - // vle B1' - // vtfmm mt4, A0, B1 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vle32.v v24, (%[b1p])\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtfmm.tvv mt4, v7, v23\n" - "addi %[b1p], %[b1p], 512\n" - - // vle A1 - // vtfmm mt4, A0', B1' || vtmv mt0 (row0 - row7) + vse - "vtmv.v.t v0, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v1, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - - "vtmv.v.t v2, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v3, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - - "vtmv.v.t v4, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v5, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - - "vtmv.v.t v6, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v7, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - - "vse32.v v0, (%[p00])\n" "addi %[p00], %[p00], 512\n" - "vtfmm.tvv mt4, v12, v28\n" - - "vtfmm.tvv mt4, v13, v29\n" - - "vtfmm.tvv mt4, v14, v30\n" - - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt4, v15, v31\n" - - // vle A1' - // vtfmm mt12, A1, B1 || vtmv mt0 (row8 - row15) + vse - "vtmv.v.t v8, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v9, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v0, v16\n" - - "vtmv.v.t v10, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v11, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - - "vtmv.v.t v12, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v13, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - - "vtmv.v.t v14, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtmv.v.t v15, %[tss0]\n" "addi %[tss0], %[tss0], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - - "vse32.v v8, (%[p00])\n" "addi %[p00], %[p00], 1536\n" - "vtfmm.tvv mt12, v4, v20\n" - - "vtfmm.tvv mt12, v5, v21\n" - - "vtfmm.tvv mt12, v6, v22\n" - - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "vtfmm.tvv mt12, v7, v23\n" - - // vle reload B0 (restore B0 pointer) - // vtfmm mt12, A1', B1' || vtmv mt4 (row0 - row7) + vse - "vtmv.v.t v16, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v17, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - - "vtmv.v.t v18, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v19, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - - "vtmv.v.t v20, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v21, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - - "vtmv.v.t v22, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v23, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - - "vse32.v v16, (%[p01])\n" "addi %[p01], %[p01], 512\n" - "vtfmm.tvv mt12, v12, v28\n" - - "vtfmm.tvv mt12, v13, v29\n" - "addi %[b0p], %[b0p], -1024\n" - - "vtfmm.tvv mt12, v14, v30\n" - - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt12, v15, v31\n" - - // vle reload B0' - // vtfmm mt8, A1, B0 || vtmv mt4 (row8 - row15) + vse - "vtmv.v.t v24, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v25, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v0, v16\n" - - "vtmv.v.t v26, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v27, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - - "vtmv.v.t v28, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v29, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - - "vtmv.v.t v30, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtmv.v.t v31, %[tss4]\n" "addi %[tss4], %[tss4], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - - "vse32.v v24, (%[p01])\n" "addi %[p01], %[p01], 1536\n" - "vtfmm.tvv mt8, v4, v20\n" - - "vtfmm.tvv mt8, v5, v21\n" - - "vtfmm.tvv mt8, v6, v22\n" - - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt8, v7, v23\n" - - // vtfmm mt8, A1', B0' || vtmv mt12 (row0 - row7) + vse - // vtmv mt12 (row8 - row15) + vse - "addi %[loop], %[tss12], 8\n" - - "vtmv.v.t v0, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v1, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - - "vtmv.v.t v2, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v3, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - - "vtmv.v.t v4, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v5, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - - "vtmv.v.t v6, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtmv.v.t v7, %[tss12]\n" "addi %[tss12], %[tss12], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - - "vse32.v v0, (%[p11])\n" "addi %[p11], %[p11], 512\n" - "vtmv.v.t v16, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v17, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - - "vtmv.v.t v18, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v19, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - - "vtmv.v.t v20, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v21, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - - "vtmv.v.t v22, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtmv.v.t v23, %[loop]\n" "addi %[loop], %[loop], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vse32.v v16, (%[p11])\n" "addi %[p11], %[p11], 1536\n" - "mv %[tss12], %[loop]\n" - - "addi %[blocks], %[blocks], -1\n" - "beqz %[blocks], 7f\n" - - "addi %[tss0], %[tss0], -16\n" - "li %[vtle_loop], 16\n" - ".Lvtle_next_mt0_%=:\n" - "vtle32 %[tss0], (%[p00])\n" - "addi %[p00], %[p00], 64\n" - "addi %[tss0], %[tss0], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_next_mt0_%=\n" - "addi %[p00], %[p00], -1024\n" - "addi %[tss0], %[tss0], -16\n" - - // Next Block - // K-Group 0 - // vle A0 - // vle B0 - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "vle32.v v0, (%[a0p])\n" - "add %[b0p], %[b0p], %[loop]\n" - "vle32.v v16, (%[b0p])\n" - - // vtmv mt8 (row0 - row7) + vse || next block vtfmm mt0, A0, B0 - // vle B1 - "vtmv.v.t v8, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v9, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt0, v0, v16\n" - - "vtmv.v.t v10, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v11, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt0, v1, v17\n" - - "vtmv.v.t v12, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v13, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - - "vtmv.v.t v14, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v15, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt0, v3, v19\n" - - "addi %[tss4], %[tss4], -16\n" - "li %[vtle_loop], 16\n" - ".Lvtle_next_mt4_%=:\n" - "vtle32 %[tss4], (%[p01])\n" - "addi %[p01], %[p01], 64\n" - "addi %[tss4], %[tss4], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_next_mt4_%=\n" - "addi %[p01], %[p01], -1024\n" - "addi %[tss4], %[tss4], -16\n" - "vse32.v v8, (%[p10])\n" "addi %[p10], %[p10], 512\n" - "vtfmm.tvv mt0, v4, v20\n" - - "vtfmm.tvv mt0, v5, v21\n" - "add %[b1p], %[b1p], %[loop]\n" - "vtfmm.tvv mt0, v6, v22\n" - - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "vtfmm.tvv mt0, v7, v23\n" - - // vtmv mt8 (row8 - row15) + vse || next block vtfmm mt4, A0, B1 - // vle A1 - "vtmv.v.t v8, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v9, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt4, v0, v24\n" - - "vtmv.v.t v10, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v11, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt4, v1, v25\n" - - "vtmv.v.t v12, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v13, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - - "vtmv.v.t v14, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v15, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtfmm.tvv mt4, v3, v27\n" - - "vse32.v v8, (%[p10])\n" "addi %[p10], %[p10], 1536\n" - "vtfmm.tvv mt4, v4, v28\n" - - "sub %[a1p], %[a1p], %[loop]\n" - "vtfmm.tvv mt4, v5, v29\n" - - "vtfmm.tvv mt4, v6, v30\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - - // preload vle A0 - // preload vle B0 - // next block vtfmm mt8, A1, B0 - "vtfmm.tvv mt4, v7, v31\n" - "addi %[a0p], %[a0p], 512\n" - "vle32.v v0, (%[a0p])\n" - "addi %[tss8], %[tss8], -16\n" - "li %[vtle_loop], 16\n" - ".Lvtle_next_mt8_%=:\n" - "vtle32 %[tss8], (%[p10])\n" - "addi %[p10], %[p10], 64\n" - "addi %[tss8], %[tss8], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_next_mt8_%=\n" - "addi %[p10], %[p10], -1024\n" - "addi %[tss8], %[tss8], -16\n" - "vtfmm.tvv mt8, v8, v16\n" - "addi %[a0p], %[a0p], 512\n" - "vtfmm.tvv mt8, v9, v17\n" - "addi %[tss12], %[tss12], -16\n" - "li %[vtle_loop], 16\n" - ".Lvtle_next_mt12_%=:\n" - "vtle32 %[tss12], (%[p11])\n" - "addi %[p11], %[p11], 64\n" - "addi %[tss12], %[tss12], 1\n" - "addi %[vtle_loop], %[vtle_loop], -1\n" - "bnez %[vtle_loop], .Lvtle_next_mt12_%=\n" - "addi %[p11], %[p11], -1024\n" - "addi %[tss12], %[tss12], -16\n" - "vtfmm.tvv mt8, v10, v18\n" - "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vle32.v v16, (%[b0p])\n" - "vtfmm.tvv mt8, v14, v22\n" - "addi %[b0p], %[b0p], 512\n" - "vtfmm.tvv mt8, v15, v23\n" - - // next block vtfmm mt12, A1, B1 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "j 9b\n" - - // Final block: compact 16-row mt8 store loop. - "7:\n" - - "vtmv.v.t v0, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v1, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v2, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v3, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v4, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v5, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v6, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v7, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - - "vse32.v v0, (%[p10])\n" - "addi %[p10], %[p10], 512\n" - "vtmv.v.t v8, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v9, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v10, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v11, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v12, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v13, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v14, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - "vtmv.v.t v15, %[tss8]\n" "addi %[tss8], %[tss8], 1\n" - - "vse32.v v8, (%[p10])\n" - "8:\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss0] "+r"(tss0), - [tss4] "+r"(tss4), - [tss8] "+r"(tss8), - [tss12] "+r"(tss12), - [p00] "+r"(p00), - [p01] "+r"(p01), - [p10] "+r"(p10), - [p11] "+r"(p11), - [blocks] "+r"(block_counter), - [loop] "=&r"(loop_counter), - [vtle_loop] "=&r"(vtle_counter) - : - [tile_dim] "r"(tile_dim), - [tk] "r"(TK) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/kernel/gemm.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/kernel/gemm.h deleted file mode 100644 index 622b9439d..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/kernel/gemm.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// gemm.h — interface for the fp32×fp32=fp32 VME GEMM kernel. - -#include - -// Read the hardware cycle counter. -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; -} - -// Reading fcsr stalls Snitch while any Spatz scoreboard entry is live. Use it -// to turn an asynchronous kernel return into an architectural completion point. -static inline void wait_spatz(void) -{ - uint32_t fcsr; - asm volatile("csrr %0, fcsr" : "=r"(fcsr) :: "memory"); -} - -// FP32 VME GEMM over TE=16 with architectural tk=1 and RES_K=8 software -// residency. [ti_lo, ti_hi) selects an even-aligned range of 16-row tiles. -void gemm_fp32(float *C, const float *Apack, const float *Bpack, - uint32_t ti_lo, uint32_t ti_hi); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/main.c deleted file mode 100644 index 689f16de1..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/main.c +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// main.c — fp32×fp32=fp32 VME GEMM functional check - -#include -#include -#include - -#include DATAHEADER -#include "kernel/gemm.c" - -// Shared L1 buffers: core 0 allocates and DMAs; the pointers live in shared -// TCDM (.data) so every core sees the same addresses. Declared at file scope -// (not on main's stack) precisely so the assignment by core 0 is visible to the -// other cores after the barrier -- the sp-fmatmul multi-core pattern. -static const float *Apack, *Bpack, *Cref; -static float *C; -static uint32_t allocation_failed; -static volatile uint32_t core_cycles[16]; - -int main(void) -{ - const uint32_t cid = snrt_cluster_core_idx(); - const uint32_t ncores = snrt_cluster_core_num(); -#ifdef SPATZ_GEMM_SINGLE_CORE - const uint32_t active_cores = 1; -#else - const uint32_t active_cores = ncores; -#endif - - const uint32_t M = gemm_l.M; - const uint32_t N = gemm_l.N; - const uint32_t K = gemm_l.K; - const uint32_t TM = 16; - const uint32_t TN = 16; - const uint32_t output_ld = N; - - // Core 0 owns the DMA engine and publishes all shared L1 pointers. - const uint32_t c_bytes = M * output_ld * sizeof(float); - const uint32_t c_ref_bytes = M * N * sizeof(float); - const uint32_t a_bytes = K * M * sizeof(float); - const uint32_t b_bytes = K * N * sizeof(float); - - if (cid == 0) { - C = (float *)snrt_l1alloc(c_bytes); - Cref = (float *)snrt_l1alloc(c_ref_bytes); - Bpack = (float *)snrt_l1alloc(b_bytes); - Apack = (float *)snrt_l1alloc(a_bytes); - allocation_failed = C == NULL || Cref == NULL || Apack == NULL || Bpack == NULL; - if (!allocation_failed) { - snrt_dma_start_1d((void *)Cref, gemm_C_dram, c_ref_bytes); - snrt_dma_start_1d((void *)Apack, gemm_Apack_dram, a_bytes); - snrt_dma_start_1d((void *)Bpack, gemm_Bpack_dram, b_bytes); - snrt_dma_wait_all(); - memset(C, 0, c_bytes); - } - } - - // Partition complete 2x2 tile blocks. This keeps the range pair-aligned and - // leaves excess cores idle when M=32 has only one row-tile pair. - const uint32_t npairs = (M / TM) / 2; - const uint32_t pairs_per_core = - (npairs + active_cores - 1) / active_cores; - uint32_t pair_lo = cid < active_cores ? cid * pairs_per_core : npairs; - uint32_t pair_hi = cid < active_cores ? pair_lo + pairs_per_core : npairs; - if (pair_hi > npairs) pair_hi = npairs; - if (pair_lo > npairs) pair_lo = npairs; - const uint32_t ti_lo = 2 * pair_lo; - const uint32_t ti_hi = 2 * pair_hi; - - // Barrier: all cores wait until the buffers are DMA'd in and pointers set. - snrt_cluster_hw_barrier(); - - if (allocation_failed) { - if (cid == 0) printf("GEMM allocation failed\n"); - return 2; - } - - if (K % 8 != 0) - return 2; - - // Synchronize the launch, but keep both barriers outside the measured - // interval. Each core measures only its own GEMM slice. - snrt_cluster_hw_barrier(); - if (cid < active_cores) { - uint32_t t0 = get_cycle(); - gemm_fp32(C, Apack, Bpack, ti_lo, ti_hi); - uint32_t t1 = get_cycle(); - core_cycles[cid] = t1 - t0; - } else { - core_cycles[cid] = 0; - } - - // Publish all per-core durations before core 0 selects the parallel - // wall-clock time, defined as the slowest core's kernel duration. - snrt_cluster_hw_barrier(); - - // Only core 0 verifies and reports. - if (cid != 0) - return 0; - - uint32_t cycles = 0; - for (uint32_t i = 0; i < active_cores; ++i) { - if (core_cycles[i] > cycles) - cycles = core_cycles[i]; - } - - enum { CE = 8 }; - const uint64_t useful_fmas = (uint64_t)M * N * K; - const uint64_t peak_fmas = - (uint64_t)cycles * CE * CE * active_cores; - const uint32_t utilization_bp = - peak_fmas != 0 ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) : 0; - - // Element-wise error check. - // fdiv.s is not supported by the Spatz FPU; use fmul.s to avoid it. - // Instead of rel_err = abs_err / den > tol, check abs_err > tol * den. - int n_err = 0; - float max_abs = 0.0f; - const float tol = 1.0e-3f; - - uint32_t row_errs[M]; - memset(row_errs, 0, sizeof(row_errs)); - for (uint32_t idx = 0; idx < M * N; idx++) { - const uint32_t row = idx / N; - const uint32_t col = idx % N; - const uint32_t tile_row = row / TM; - const uint32_t tile_col = col / TN; - const uint32_t row_in_tile = row % TM; - const uint32_t col_in_tile = col % TN; - const uint32_t packed_idx = - (tile_row * (N / TN) + tile_col) * (TM * TN) + - row_in_tile * TN + col_in_tile; - float diff = C[packed_idx] - Cref[idx]; - float abs_err = diff < 0.0f ? -diff : diff; - float den = Cref[idx] < 0.0f ? -Cref[idx] : Cref[idx]; - if (den < 1.0e-9f) den = 1.0e-9f; - if (abs_err > max_abs) max_abs = abs_err; - if (abs_err > tol && abs_err > tol * den) { - row_errs[idx / N]++; - n_err++; - } - } - for (uint32_t r = 0; r < M; r++) { - if (row_errs[r]) - printf("row %u: %u errors\n", (unsigned)r, (unsigned)row_errs[r]); - } - - printf("\n=== VME fp32 GEMM %ux%ux%u (TM=%u TN=%u, %u core%s) ===\n", - (unsigned)M, (unsigned)N, (unsigned)K, - (unsigned)TM, (unsigned)TN, - (unsigned)active_cores, active_cores == 1 ? "" : "s"); - printf("Cycles: %u\n", (unsigned)cycles); - printf("Utilization: %u.%02u%%\n", (unsigned)(utilization_bp / 100), - (unsigned)(utilization_bp % 100)); - printf("MaxAbsErr: 0x%08x\n", *(unsigned *)&max_abs); - printf("Errors: %d\n", n_err); - printf("Status: %s\n", n_err == 0 ? "PASS" : "FAIL"); - - // No trailing barrier: the non-zero cores have already returned, so an extra - // barrier here would have no partner and deadlock. - return n_err == 0 ? 0 : 1; -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/script/gen_data.py deleted file mode 100644 index 637655fc8..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/store-interleave/script/gen_data.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_gemm_layer gemm_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - f" .TM = {TM},", - f" .TN = {TN},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float gemm_C_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = gemm_Apack_dram + (mt*(K/4)+g)*64.", - f'static float gemm_Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = gemm_Bpack_dram + (nt*(K/4)+g)*64.", - f'static float gemm_Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/vsetvl/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/vsetvl/CMakeLists.txt deleted file mode 100644 index 5a6599851..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/rolled/vsetvl/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(rolled-vsetvl "${CMAKE_CURRENT_SOURCE_DIR}") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/CMakeLists.txt deleted file mode 100644 index b660ea961..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(unrolled-store-afterK "${CMAKE_CURRENT_SOURCE_DIR}" "64x64x64") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/kernel/matmul.c deleted file mode 100644 index dd4c03ee2..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/kernel/matmul.c +++ /dev/null @@ -1,2504 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. - -#include "matmul.h" - -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) -{ - const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); - uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); - uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; - uintptr_t c00 = (uintptr_t)(C); - uintptr_t c01 = c00 + operand_bytes; - uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); - uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t loop_counter; - uintptr_t vl_counter; - const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - - asm volatile( - - // Block 1, K-Group 1 - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=1, twiden=1 - "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE - // we dont need preload - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - - // Block 2, K-Group 1 - // vtfmm mt0 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v1, v17\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt4 - "vtzero mt4\n" - "vtfmm.tvv mt4, v0, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" - - // vtfmm mt12 - "vtzero mt12\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // vtfmm mt8 - "vtzero mt8\n" - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 2 - // vtfmm mt0 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "add %[a0p], %[a0p], %[loop]\n" - "add %[a1p], %[a1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - - // Block 3, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[loop], x0, %[loop]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // Block 3, K-Group 2 - // vtfmm mt0 - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - - // Block 4, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // Block 4, K-Group 2 - // vtfmm mt0 - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - // [blocks] "+r"(block_counter), - // [cols] "+&r"(col_counter), - [loop] "=&r"(loop_counter), - [vl] "=&r"(vl_counter) - : - [c_stride] "r"(c_stride), - [operand_bytes] "r"(operand_bytes), - [mtype] "r"(matrix_mtype), - [vtype] "r"(matrix_vtype) - // [rewind_b0_bytes] "r"(rewind_b0_bytes), - // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/script/gemm_64_64_64.json deleted file mode 100644 index 3a26d4cde..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/script/gemm_64_64_64.json +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. -{ - kernel: "VME_GEMM", - M: 64, - N: 64, - K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) - seed: 42 -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/script/gen_data.py deleted file mode 100644 index efa468aa3..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_afterK/K64/script/gen_data.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = Apack_dram + (mt*(K/4)+g)*64.", - f'static float Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = Bpack_dram + (nt*(K/4)+g)*64.", - f'static float Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/CMakeLists.txt deleted file mode 100644 index f4087725b..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(unrolled-store-interleave "${CMAKE_CURRENT_SOURCE_DIR}" "64x64x64") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/kernel/matmul.c deleted file mode 100644 index ad8339f9b..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/kernel/matmul.c +++ /dev/null @@ -1,2774 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. - -#include "matmul.h" - -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) -{ - const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); - uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); - uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; - uintptr_t c00 = (uintptr_t)(C); - uintptr_t c01 = c00 + operand_bytes; - uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); - uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t loop_counter; - uintptr_t vl_counter; - const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - - asm volatile( - // ".Lmatmul_block_%=:\n" - // K-Group 1 - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=1, twiden=1 - "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE - // we dont need preload - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // "9:\n" - // "mv %[loop], %[middle_k_groups]\n" - // "beqz %[loop], 3f\n" - // "1:\n" - - // K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group 7, 8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v9, v25\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" - "vle32.v v13, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v29, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v15, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v13, v29\n" - "vle32.v v31, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Complete the block before changing any A/B/C panel pointer. The - // former cross-block path deferred this mt8 store while beginning the - // next block, but its FP32 register preload was incomplete. - - // // block transition - // "addi %[blocks], %[blocks], -1\n" - // "beqz %[blocks], 8f\n" - - // // Advance horizontally until the N panel is exhausted. At the row - // // boundary, advance A by two tiles and rewind B to its first pair. - // "addi %[cols], %[cols], -1\n" - // "beqz %[cols], 4f\n" - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - // Flattened successor block. The previous mt8 is fully stored before - // this block reuses it, so TSS starts at mt0,row0. - // K-Group 1 - // vtfmm mt0 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtzero mt0\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v13, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - // "j 9b\n" - // "9:\n" - // "mv %[loop], %[middle_k_groups]\n" - // "beqz %[loop], 3f\n" - // "1:\n" - - // K-Group 2 - // vtfmm mt0 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group 7, 8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v9, v25\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" - "vle32.v v13, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v29, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v15, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v13, v29\n" - "vle32.v v31, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Final block: compact 16-row mt8 store loop. - // "7:\n" - // "li %[loop], 16\n" - // "2:\n" - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 2b\n" - // "8:\n" - - // Move from the top-right 32x32 block to the bottom-left block. - // A advances by one 32-row panel, B rewinds to its first column - // panel, and C advances diagonally from (0,32) to (32,0). - "sub %[loop], %[b1p], %[b0p]\n" - "add %[a0p], %[a0p], %[loop]\n" - "add %[a1p], %[a1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[loop], x0, %[loop]\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - - // ".Lmatmul_block_%=:\n" - "mv %[tss], x0\n" - // K-Group 1 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // "9:\n" - // "mv %[loop], %[middle_k_groups]\n" - // "beqz %[loop], 3f\n" - // "1:\n" - - // K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group 7, 8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v9, v25\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" - "vle32.v v13, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v29, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v15, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v13, v29\n" - "vle32.v v31, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Complete the block before changing any A/B/C panel pointer. The - // former cross-block path deferred this mt8 store while beginning the - // next block, but its FP32 register preload was incomplete. - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // // block transition - "vtzero mt0\n" - // "addi %[blocks], %[blocks], -1\n" - // "beqz %[blocks], 8f\n" - - // // Advance horizontally until the N panel is exhausted. At the row - // // boundary, advance A by two tiles and rewind B to its first pair. - // "addi %[cols], %[cols], -1\n" - // "beqz %[cols], 4f\n" - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - // "j 5f\n" - // "4:\n" - // "sub %[cols], %[b1p], %[b0p]\n" - // "add %[a0p], %[a0p], %[cols]\n" - // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" - // "addi %[loop], %[loop], -1\n" - // "6:\n" - // "sub %[b0p], %[b0p], %[cols]\n" - // "sub %[b1p], %[b1p], %[cols]\n" - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" - // "slli %[loop], %[c_stride], 4\n" - // "addi %[loop], %[loop], -128\n" - // "sub %[loop], x0, %[loop]\n" - - // "5:\n" - // "sub %[c00], %[c00], %[loop]\n" - // "sub %[c01], %[c01], %[loop]\n" - // "sub %[c10], %[c10], %[loop]\n" - // "sub %[c11], %[c11], %[loop]\n" - // "j .Lmatmul_block_%=\n" - - // Flattened successor block. The previous mt8 is fully stored before - // this block reuses it, so TSS starts at mt0,row0. - "mv %[tss], x0\n" - // K-Group 1 - // vtfmm mt0 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v0, v24\n" - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - // "j 9b\n" - // "9:\n" - // "mv %[loop], %[middle_k_groups]\n" - // "beqz %[loop], 3f\n" - // "1:\n" - - // K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 1b\n" - // "3:\n" - - // K-Group 7, 8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v8, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v24, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v9, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v25, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v10, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v26, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v11, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vle32.v v27, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v12, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v9, v25\n" - "vle32.v v28, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v10, v26\n" - "vle32.v v13, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v29, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v11, v27\n" - "vle32.v v14, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vle32.v v30, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v15, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v13, v29\n" - "vle32.v v31, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Final block: compact 16-row mt8 store loop. - // "7:\n" - // "li %[loop], 16\n" - // "2:\n" - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 2b\n" - // "8:\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - // [blocks] "+r"(block_counter), - // [cols] "+&r"(col_counter), - [loop] "=&r"(loop_counter), - [vl] "=&r"(vl_counter) - : - [c_stride] "r"(c_stride), - [operand_bytes] "r"(operand_bytes), - [mtype] "r"(matrix_mtype), - [vtype] "r"(matrix_vtype) - // [rewind_b0_bytes] "r"(rewind_b0_bytes), - // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/script/gemm_64_64_64.json deleted file mode 100644 index 3a26d4cde..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/script/gemm_64_64_64.json +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. -{ - kernel: "VME_GEMM", - M: 64, - N: 64, - K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) - seed: 42 -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/script/gen_data.py deleted file mode 100644 index efa468aa3..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/store_interleave/K64/script/gen_data.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = Apack_dram + (mt*(K/4)+g)*64.", - f'static float Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = Bpack_dram + (nt*(K/4)+g)*64.", - f'static float Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/CMakeLists.txt deleted file mode 100644 index 99f9db7d6..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(unrolled-vsetvl "${CMAKE_CURRENT_SOURCE_DIR}" "64x64x128") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/script/gemm_64_64_64.json deleted file mode 100644 index 3a26d4cde..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/script/gemm_64_64_64.json +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. -{ - kernel: "VME_GEMM", - M: 64, - N: 64, - K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) - seed: 42 -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/script/gen_data.py deleted file mode 100644 index efa468aa3..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K128/script/gen_data.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = Apack_dram + (mt*(K/4)+g)*64.", - f'static float Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = Bpack_dram + (nt*(K/4)+g)*64.", - f'static float Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/CMakeLists.txt deleted file mode 100644 index 9921325ef..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(unrolled-vsetvl "${CMAKE_CURRENT_SOURCE_DIR}" "64x64x256") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/data/layer.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/data/layer.h deleted file mode 100644 index 8d340d758..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/data/layer.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2024 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include - -/** - * @brief VME matmul layer configuration. - * - * Describes a C = A * B problem with dimensions M x N x K. - */ -typedef struct { - uint32_t M; - uint32_t N; - uint32_t K; -} vme_matmul_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/main.c deleted file mode 100644 index 5d067c3dc..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/main.c +++ /dev/null @@ -1,269 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// main.c - fp32 x fp32 = fp32 VME GEMM functional check - -#include -#include -#include - -#include DATAHEADER -#include "kernel/matmul.c" - -enum { - TCDM_BYTES = 256 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = 4 * CHUNK_SIZE, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = 2, - L1_BYTES = BUFFER_BYTES, -}; - -_Static_assert(L1_BYTES <= TCDM_BYTES, - "one M64xN64xK256 TEW32 buffer exceeds the 256 KiB TCDM"); - -static const float zero_tile_dram[TILE_INPUTS] - __attribute__((section(".dram"), aligned(64))) = {0}; - -static uint8_t *l1_base; -static float *partials; -static float *c_out; - -static inline uint32_t min_u32(uint32_t a, uint32_t b) { - return a < b ? a : b; -} - -static inline uint32_t chunk_extent(uint32_t total, uint32_t block, - uint32_t chunk) { - const uint32_t offset = block * chunk; - return offset < total ? min_u32(chunk, total - offset) : 0; -} - -static inline uint32_t float_bits(float value) { - union { - float f; - uint32_t u; - } bits = {.f = value}; - return bits.u; -} - -static inline uint32_t cycle_count(void) { - uint32_t cycles; - asm volatile("csrr %0, mcycle" : "=r"(cycles)); - return cycles; -} -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { - if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { - return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } - - const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( - dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); - if (valid_tiles == 0 || valid_k == 0) - return clear_tid; - - const size_t copy_bytes = valid_k * TE * sizeof(float); - const size_t src_stride = src_tile_k * TE * sizeof(float); - // The assembly kernel advances between packed tiles by 16 * valid_k - // elements. Keep the L1 panel compact even though each buffer reserves - // space for the maximum K_CHUNK. - return snrt_dma_start_2d(dst, src, copy_bytes, copy_bytes, src_stride, - valid_tiles); -} - -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint32_t buffer, uint32_t mb, uint32_t nb, - uint32_t kb) { - uint8_t *base = l1_base + buffer * BUFFER_BYTES; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); -} - -static inline float partial_at(const float *partial, uint32_t row, - uint32_t col) { - return (float)partial[(size_t)row * N_CHUNK + col]; -} - -int main(void) { - if (snrt_cluster_core_idx() != 0) - return 0; - - const uint32_t mb_count = (matmul_l.M + M_CHUNK - 1) / M_CHUNK; - const uint32_t nb_count = (matmul_l.N + N_CHUNK - 1) / N_CHUNK; - const uint32_t kb_count = (matmul_l.K + K_CHUNK - 1) / K_CHUNK; - const uint32_t jobs = mb_count * nb_count; - const uint32_t iterations = jobs * kb_count; - - l1_base = (uint8_t *)snrt_l1alloc(L1_BYTES); - partials = (float *)snrt_l3alloc((size_t)iterations * C_BYTES); - c_out = (float *)snrt_l3alloc((size_t)matmul_l.M * matmul_l.N * - sizeof(float)); - if (l1_base == 0 || partials == 0 || c_out == 0) { - printf("fp32 matmul allocation failure\n"); - return 2; - } - - snrt_dma_wait(fill_buffer(0, 0, 0, 0)); - - uint32_t start_cycle = 0; - uint32_t end_cycle = 0; - for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - uint8_t *base = l1_base; - uint32_t fcsr; - - if (iter == 0) - start_cycle = cycle_count(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); - end_cycle = cycle_count(); - - snrt_dma_wait(snrt_dma_start_1d( - partials + (size_t)iter * C_ELEMENTS, base + C_OFFSET, C_BYTES)); - - const uint32_t next_iter = iter + 1; - if (next_iter < iterations) { - const uint32_t next_job = next_iter / kb_count; - snrt_dma_wait(fill_buffer(0, next_job / nb_count, - next_job % nb_count, - next_iter % kb_count)); - } - } - snrt_dma_wait_all(); - - for (uint32_t row = 0; row < matmul_l.M; ++row) { - for (uint32_t col = 0; col < matmul_l.N; ++col) { - const uint32_t mb = row / M_CHUNK; - const uint32_t nb = col / N_CHUNK; - const uint32_t job = mb * nb_count + nb; - const uint32_t local_row = row % M_CHUNK; - const uint32_t local_col = col % N_CHUNK; - float sum = 0.0f; - for (uint32_t kb = 0; kb < kb_count; ++kb) { - const float *partial = partials + - (size_t)(job * kb_count + kb) * C_ELEMENTS; - sum += partial_at(partial, local_row, local_col); - } - c_out[(size_t)row * matmul_l.N + col] = sum; - } - } - - uint32_t errors = 0; - float max_abs = 0.0f; - for (uint32_t row = 0; row < matmul_l.M; ++row) { - uint32_t row_errors = 0; - for (uint32_t col = 0; col < matmul_l.N; ++col) { - const size_t index = (size_t)row * matmul_l.N + col; - const float actual = c_out[index]; - const float expected = Cref_dram[index]; - const float delta = actual > expected ? actual - expected - : expected - actual; - const float tolerance = 0.05f + 0.02f * - (expected < 0.0f ? -expected : expected); - if (delta > max_abs) - max_abs = delta; - if (delta > tolerance) { - if (row_errors == 0) - printf("row %u: errors at cols ", row); - else - printf(","); - printf("%u", col); - ++row_errors; - ++errors; - } - } - if (row_errors != 0) - printf(" (%u errors)\n", row_errors); - } - - const uint32_t cycles = end_cycle - start_cycle; - const uint64_t useful_fmas = - (uint64_t)matmul_l.M * matmul_l.N * matmul_l.K; - const uint64_t peak_fmas = (uint64_t)cycles * CE * CE; - const uint32_t utilization_bp = - peak_fmas != 0 - ? (uint32_t)((useful_fmas * 10000u) / peak_fmas) - : 0; - - printf("\n=== TEW32 FP32 matmul %ux%ux%u " - "(M%uxN%uxK%u chunks, %u buffers, 1 core) ===\n", - matmul_l.M, matmul_l.N, matmul_l.K, - M_CHUNK, N_CHUNK, K_CHUNK, L1_BUFFER_COUNT); - printf("Cycles: %u\n", cycles); - printf("Utilization: %u.%02u%%\n", utilization_bp / 100, - utilization_bp % 100); - printf("MaxAbsErr: 0x%08x\n", float_bits(max_abs)); - printf("Errors: %u\n", errors); - printf("Status: %s\n", errors ? "FAIL" : "PASS"); - return errors ? 1 : 0; -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/script/gen_data.py deleted file mode 100644 index efa468aa3..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K256/script/gen_data.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = Apack_dram + (mt*(K/4)+g)*64.", - f'static float Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = Bpack_dram + (nt*(K/4)+g)*64.", - f'static float Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/CMakeLists.txt deleted file mode 100644 index 3c50bb5b6..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_fp32_vtse_variant(unrolled-vsetvl "${CMAKE_CURRENT_SOURCE_DIR}" "64x64x64") diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/data/layer.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/data/layer.h deleted file mode 100644 index 8d340d758..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/data/layer.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2024 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#include - -/** - * @brief VME matmul layer configuration. - * - * Describes a C = A * B problem with dimensions M x N x K. - */ -typedef struct { - uint32_t M; - uint32_t N; - uint32_t K; -} vme_matmul_layer; diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul copy.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul copy.c deleted file mode 100644 index 81fbd578f..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul copy.c +++ /dev/null @@ -1,2287 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// fp32 optimal GEMM kernel for TE=16, CE=8, and four accumulator tiles. - -#include "matmul.h" - -__attribute__((noinline, aligned(64))) void matmul_fp32( - const float *Apack, const float *Bpack, float *C, - uint32_t M, uint32_t N, uint32_t K) -{ - // const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - // const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)Apack; - uintptr_t a1p = (uintptr_t)(Apack + tile_stride); - uintptr_t b0p = (uintptr_t)Bpack; - uintptr_t b1p = (uintptr_t)(Bpack + tile_stride); - uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; - uintptr_t c00 = (uintptr_t)(C); - uintptr_t c01 = c00 + operand_bytes; - uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); - uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - // uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - // uintptr_t col_counter = (uintptr_t)col_blocks; - uintptr_t loop_counter; - uintptr_t vl_counter; - const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - - asm volatile( - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - // Block 1, K-Group 1 - "vtzero mt0\n" - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 1, K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 1, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 1, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 1, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 1, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - - // Block 2, K-Group 1 - // vtfmm mt0 - "vtzero mt0\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v10, v18\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[c00], %[c00], %[loop]\n" - "vtfmm.tvv mt8, v11, v19\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v15, v31\n" - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Drain Block 2 mt8 alongside Block 3's other tiles. - - "sub %[loop], %[b1p], %[b0p]\n" - "add %[a0p], %[a0p], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "add %[a1p], %[a1p], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[loop], x0, %[loop]\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - // c10 still addresses Block 2 mt8 until its deferred store finishes. - "sub %[c11], %[c11], %[loop]\n" - - "lui %[tss], 0x40000\n" - // Block 3, K-Group 1 - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "sub %[c10], %[c10], %[loop]\n" - "vtzero mt8\n" - // Block 3 mt8 can begin only after the old mt8 is fully stored. - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 3, K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 3, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 3, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 3, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 3, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "nop\n" - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Defer the old block's mt8 store and interleave its 16 beats with - // Block 4 K-group 1, matching the Block 1 -> Block 2 schedule. - - // // block transition - "vtzero mt0\n" - // "addi %[blocks], %[blocks], -1\n" - // "beqz %[blocks], 8f\n" - - // // Advance horizontally until the N panel is exhausted. At the row - // // boundary, advance A by two tiles and rewind B to its first pair. - // "addi %[cols], %[cols], -1\n" - // "beqz %[cols], 4f\n" - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - - "mv %[tss], x0\n" - // Block 4, K-Group 1: mirror the Block 2 handoff schedule. - // The prior K-group has already prepared v1..v7 and v17..v23. - // vtfmm mt0 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtzero mt0\n" - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtfmm.tvv mt0, v1, v17\n" - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtzero mt4\n" - - // vtfmm mt4 - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "vtfmm.tvv mt4, v0, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v5, v29\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" - "vtzero mt12\n" - - // vtfmm mt12; the last four deferred mt8 stores happen here. - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v13, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtzero mt8\n" - - // vtfmm mt8: eight FMAs follow the final deferred store. - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 4, K-Group 2 - // vtfmm mt0 - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 4, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 4, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 4, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 4, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 7+8 - "mv %[tss], x0\n" - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "nop\n" - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "nop\n" - "vtfmm.tvv mt4, v0, v16\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v17\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" - "vtfmm.tvv mt12, v0, v16\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v1, v17\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" - "vtfmm.tvv mt8, v0, v16\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v1, v17\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // Final block: compact 16-row mt8 store loop. - // "7:\n" - // "li %[loop], 16\n" - // "2:\n" - "lui %[tss], 0x40000\n" - "nop\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // "addi %[loop], %[loop], -1\n" - // "bnez %[loop], 2b\n" - // "8:\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - // [blocks] "+r"(block_counter), - // [cols] "+&r"(col_counter), - [loop] "=&r"(loop_counter), - [vl] "=&r"(vl_counter) - : - [c_stride] "r"(c_stride), - [mtype] "r"(matrix_mtype), - [vtype] "r"(matrix_vtype) - // [rewind_b0_bytes] "r"(rewind_b0_bytes), - // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul.c deleted file mode 100644 index dd4c03ee2..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul.c +++ /dev/null @@ -1,2504 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -// fp32 optimal kernel for TE=16, CE=8, and four accumulator tiles. - -#include "matmul.h" - -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) -{ - const uint32_t tile_stride = TE * K; - uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); - uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); - uintptr_t tss; - const uintptr_t operand_bytes = sizeof(float) * TE; - uintptr_t c00 = (uintptr_t)(C); - uintptr_t c01 = c00 + operand_bytes; - uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); - uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t loop_counter; - uintptr_t vl_counter; - const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); - const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; - const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - - asm volatile( - - // Block 1, K-Group 1 - "vsetvli %[loop], x0, e32, m8, ta, ma\n" - "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=1, twiden=1 - "msettn x0, %[loop]\n" // msetmtype resets tn, so set full tn=TE - // we dont need preload - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 2 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 1, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - - // Block 2, K-Group 1 - // vtfmm mt0 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" - "vtfmm.tvv mt0, v0, v16\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v1, v17\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v2, v18\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v3, v19\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v4, v20\n" - "vtfmm.tvv mt0, v5, v21\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v6, v22\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v7, v23\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt4 - "vtzero mt4\n" - "vtfmm.tvv mt4, v0, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v1, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v2, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v3, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v28\n" - "vtfmm.tvv mt4, v5, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v6, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v31\n" - - // vtfmm mt12 - "vtzero mt12\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // vtfmm mt8 - "vtzero mt8\n" - "vtfmm.tvv mt8, v8, v16\n" - "vtfmm.tvv mt8, v9, v17\n" - "vtfmm.tvv mt8, v10, v18\n" - "vtfmm.tvv mt8, v11, v19\n" - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "vtfmm.tvv mt8, v12, v20\n" - "vtfmm.tvv mt8, v13, v21\n" - "vtfmm.tvv mt8, v14, v22\n" - "vtfmm.tvv mt8, v15, v23\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" - "msettn x0, %[vl]\n" - - // Block 2, K-Group 2 - // vtfmm mt0 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 2, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "add %[a0p], %[a0p], %[loop]\n" - "add %[a1p], %[a1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - "sub %[b0p], %[b0p], %[loop]\n" - "sub %[b1p], %[b1p], %[loop]\n" - - // Block 3, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "sub %[loop], x0, %[loop]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // Block 3, K-Group 2 - // vtfmm mt0 - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 3, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "sub %[loop], %[b1p], %[b0p]\n" - "sub %[a0p], %[a0p], %[loop]\n" - "sub %[a1p], %[a1p], %[loop]\n" - "add %[b0p], %[b0p], %[loop]\n" - "add %[b1p], %[b1p], %[loop]\n" - - // Block 4, K-Group 1 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "lui %[tss], 0x40000\n" // mt8 - "vtzero mt0\n" - - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vtzero mt4\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtzero mt12\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtzero mt8\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "slli %[loop], %[c_stride], 4\n" - "addi %[loop], %[loop], -128\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "sub %[c00], %[c00], %[loop]\n" - "sub %[c01], %[c01], %[loop]\n" - "sub %[c10], %[c10], %[loop]\n" - "sub %[c11], %[c11], %[loop]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // Block 4, K-Group 2 - // vtfmm mt0 - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 3 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 4 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 5 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vle32.v v0, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vle32.v v16, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vle32.v v1, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - "vle32.v v17, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vle32.v v2, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v9, v25\n" - "vle32.v v18, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v10, v26\n" - "vle32.v v3, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v11, v27\n" - "vle32.v v19, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v12, v28\n" - "vle32.v v4, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vle32.v v20, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v14, v30\n" - "vle32.v v5, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 6 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vle32.v v21, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vle32.v v6, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vle32.v v22, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vle32.v v7, (%[a0p])\n" "add %[a0p], %[a0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vle32.v v23, (%[b0p])\n" "add %[b0p], %[b0p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vle32.v v24, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vle32.v v25, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vle32.v v26, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - - // vtfmm mt4 - "vtfmm.tvv mt4, v0, v24\n" // mt4 += v0*v24 - "vle32.v v27, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v1, v25\n" // mt4 += v1*v25 - "vle32.v v28, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v2, v26\n" // mt4 += v2*v26 - "vle32.v v29, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v3, v27\n" // mt4 += v3*v27 - "vle32.v v30, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v4, v28\n" // mt4 += v4*v28 - "vle32.v v31, (%[b1p])\n" "add %[b1p], %[b1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v5, v29\n" // mt4 += v5*v29 - "vle32.v v8, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v6, v30\n" // mt4 += v6*v30 - "vle32.v v9, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt4, v7, v31\n" // mt4 += v7*v31 - "vle32.v v10, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - - // vtfmm mt8 - "vtfmm.tvv mt8, v8, v16\n" // mt8 += v8*v16 - "vle32.v v11, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v9, v17\n" // mt8 += v9*v17 - "vle32.v v12, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vle32.v v13, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v10, v18\n" // mt8 += v10*v18 - "vle32.v v14, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v11, v19\n" // mt8 += v11*v19 - "vle32.v v15, (%[a1p])\n" "add %[a1p], %[a1p], %[operand_bytes]\n" - "vtfmm.tvv mt8, v12, v20\n" // mt8 += v12*v20 - "vtfmm.tvv mt8, v13, v21\n" // mt8 += v13*v21 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v14, v22\n" // mt8 += v14*v22 - "vtfmm.tvv mt8, v15, v23\n" // mt8 += v15*v23 - - // vtfmm mt12 - "vtfmm.tvv mt12, v8, v24\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtfmm.tvv mt12, v12, v28\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtfmm.tvv mt12, v15, v31\n" - - // Block 4, K-Group 7+8 - // vtfmm mt0 - "vtfmm.tvv mt0, v0, v16\n" // mt0 += v0*v16 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a0p])\n" "addi %[a0p], %[a0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v1, v17\n" // mt0 += v1*v17 - "vtfmm.tvv mt0, v2, v18\n" // mt0 += v2*v18 - "vtfmm.tvv mt0, v3, v19\n" // mt0 += v3*v19 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v4, v20\n" // mt0 += v4*v20 - "vtfmm.tvv mt0, v5, v21\n" // mt0 += v5*v21 - "vtfmm.tvv mt0, v6, v22\n" // mt0 += v6*v22 - "vtfmm.tvv mt0, v7, v23\n" // mt0 += v7*v23 - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt0, v8, v24\n" - "vtfmm.tvv mt0, v9, v25\n" - "vtfmm.tvv mt0, v10, v26\n" - "vtfmm.tvv mt0, v11, v27\n" - "vtfmm.tvv mt0, v12, v28\n" - "vtfmm.tvv mt0, v13, v29\n" - "vtfmm.tvv mt0, v14, v30\n" - "vtfmm.tvv mt0, v15, v31\n" - - // vtfmm mt4 - // vtse mt0 - "mv %[tss], x0\n" // mt0 - "vtfmm.tvv mt4, v0, v16\n" - "vtfmm.tvv mt4, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b1p])\n" "addi %[b1p], %[b1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v2, v18\n" - "vtfmm.tvv mt4, v3, v19\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v4, v20\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v5, v21\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v6, v22\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v7, v23\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v8, v24\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v9, v25\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v10, v26\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v11, v27\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v12, v28\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v13, v29\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v0, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt4, v14, v30\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt4, v15, v31\n" - "vtse32 %[tss], (%[c00])\n" "add %[c00], %[c00], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt12 - // vtse mt4 - "lui %[tss], 0x20000\n" // mt4 - "vtfmm.tvv mt12, v0, v16\n" - "vtfmm.tvv mt12, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v8, (%[a1p])\n" "addi %[a1p], %[a1p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v2, v18\n" - "vtfmm.tvv mt12, v3, v19\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v4, v20\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v5, v21\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v6, v22\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v7, v23\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v8, v24\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v9, v25\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v10, v26\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v11, v27\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v12, v28\n" - "addi %[b0p], %[b0p], -1024\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v16, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt12, v13, v29\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v14, v30\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt12, v15, v31\n" - "vtse32 %[tss], (%[c01])\n" "add %[c01], %[c01], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - // vtfmm mt8 - // vtse mt12 - "lui %[tss], 0x60000\n" // mt12 - "vtfmm.tvv mt8, v0, v16\n" - "vtfmm.tvv mt8, v1, v17\n" - "vsetvli %[vl], x0, e32, m8, ta, ma\n" - "vle32.v v24, (%[b0p])\n" "addi %[b0p], %[b0p], 512\n" - "msetmtype %[mtype], %[vtype]\n" "msettn x0, %[vl]\n" - "vtfmm.tvv mt8, v2, v18\n" - "vtfmm.tvv mt8, v3, v19\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v4, v20\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v5, v21\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v6, v22\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v7, v23\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v8, v24\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v9, v25\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v10, v26\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v11, v27\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v12, v28\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v13, v29\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v14, v30\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtfmm.tvv mt8, v15, v31\n" - "vtse32 %[tss], (%[c11])\n" "add %[c11], %[c11], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - "lui %[tss], 0x40000\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - "vtse32 %[tss], (%[c10])\n" "add %[c10], %[c10], %[c_stride]\n" "addi %[tss], %[tss], 1\n" - - : - [a0p] "+r"(a0p), - [a1p] "+r"(a1p), - [b0p] "+r"(b0p), - [b1p] "+r"(b1p), - [tss] "=&r"(tss), - [c00] "+r"(c00), - [c01] "+r"(c01), - [c10] "+r"(c10), - [c11] "+r"(c11), - // [blocks] "+r"(block_counter), - // [cols] "+&r"(col_counter), - [loop] "=&r"(loop_counter), - [vl] "=&r"(vl_counter) - : - [c_stride] "r"(c_stride), - [operand_bytes] "r"(operand_bytes), - [mtype] "r"(matrix_mtype), - [vtype] "r"(matrix_vtype) - // [rewind_b0_bytes] "r"(rewind_b0_bytes), - // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) - : "memory"); -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul.h deleted file mode 100644 index 099df1f0a..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/kernel/matmul.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Author: Pei-Yu Lin, ETH Zürich - -#pragma once - -#include - -enum { - CE = 8, - TE = 16, - CHUNK_SIZE = 64, - BLOCK_DIM = 2 * TE, -}; - -// FP16 VME matmul over TE=16 with architectural tk=2. C is a padded -// M-by-CHUNK_SIZE L1 panel; M and N must not exceed 64. -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/script/gemm_64_64_64.json b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/script/gemm_64_64_64.json deleted file mode 100644 index 3a26d4cde..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/script/gemm_64_64_64.json +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2026 ETH Zurich and University of Bologna. -// Licensed under the Apache License, Version 2.0, see LICENSE for details. -// SPDX-License-Identifier: Apache-2.0 -// -// Author: Pei-Yu Lin, ETH Zürich -// -// Configuration for a 64×64×64 VME GEMM benchmark. -{ - kernel: "VME_GEMM", - M: 64, - N: 64, - K: 64, - TM: 8, - TN: 8, - // Seed for reproducible random data (numpy / torch) - seed: 42 -} diff --git a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/script/gen_data.py deleted file mode 100644 index efa468aa3..000000000 --- a/sw/vme/apps/tew32/fp32-matmul-vtle-vse/unrolled/vsetvl/K64/script/gen_data.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026 ETH Zurich and University of Bologna. -# Licensed under the Apache License, Version 2.0, see LICENSE for details. -# SPDX-License-Identifier: Apache-2.0 -# -# Author: Pei-Yu Lin -# -# -# Usage: -# python3 script/gen_data.py -c script/gemm_64_64_64.json -# -# Output: -# data/data_{M}_{N}_{K}.h -# -# The config JSON must provide: M, N, K. -# Optional fields: TM (default 16), TN (default 16), seed (default 42). - -import argparse -import pathlib - -import hjson -import numpy as np -import torch - -def array_to_cstr_f32(arr: np.ndarray) -> str: - """Flat float32 array → C brace-initializer with 'f' suffix.""" - return "{" + ", ".join(repr(float(x)) + "f" for x in arr.flat) + "}" - -def main(): - parser = argparse.ArgumentParser( - description="Generate fp32 VME GEMM benchmark data header" - ) - parser.add_argument( - "-c", "--cfg", - type=pathlib.Path, - required=True, - help="JSON/HJSON config file (e.g. script/gemm_64_64_64.json)", - ) - parser.add_argument("-v", "--verbose", action="store_true") - args = parser.parse_args() - - with args.cfg.open() as f: - param = hjson.loads(f.read()) - - M = int(param["M"]) - N = int(param["N"]) - K = int(param["K"]) - TM = int(param.get("TM", 16)) - TN = int(param.get("TN", 16)) - seed = int(param.get("seed", 42)) - - assert M % (2 * TM) == 0, f"M={M} must be a multiple of 2*TM={2*TM}" - assert N % (2 * TN) == 0, f"N={N} must be a multiple of 2*TN={2*TN}" - - torch.manual_seed(seed) - np.random.seed(seed) - - A = torch.randn((M, K), dtype=torch.float32).numpy() - B = torch.randn((K, N), dtype=torch.float32).numpy() - C = (torch.tensor(A) @ torch.tensor(B)).numpy() - - # Contiguous k-fused-quad packing for the kcontig kernels. - TK = 4 - TW16 = 16 - assert K % TK == 0 - assert M % TW16 == 0 and N % TW16 == 0 - # A_pack[m_tile, k_group, slice, col] = A[m_tile*TW + col, 4*kg + slice] - A_pack = (A.reshape(M // TW16, TW16, K // TK, TK) # (mt, col, kg, slice) - .transpose(0, 2, 3, 1) # (mt, kg, slice, col) - .copy()) - B_pack = (B.reshape(K // TK, TK, N // TW16, TW16) # (kg, slice, nt, col) - .transpose(2, 0, 1, 3) # (nt, kg, slice, col) - .copy()) - if args.verbose: - print(f"A {A.shape} min={A.min():.4f} max={A.max():.4f}") - print(f"B {B.shape} min={B.min():.4f} max={B.max():.4f}") - print(f"C {C.shape} min={C.min():.4f} max={C.max():.4f}") - - lines = [ - "// Copyright 2026 ETH Zurich and University of Bologna.", - "// Licensed under the Apache License, Version 2.0, see LICENSE for details.", - "// SPDX-License-Identifier: Apache-2.0", - "//", - "// Author: Pei-Yu Lin ", - "//", - "// Auto-generated by script/gen_data.py — do not edit by hand.", - "", - '#include "layer.h"', - "", - "const vme_matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", - "};", - "", - "// C[M*N] row-major fp32 golden reference, C = A*B", - f'static float Cref_dram[{M}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(C) + ";", - "", - "", - "// Contiguous k-fused-quad packing (for the kcontig / LMUL=4 kernel).", - "// A_pack[m_tile][k_group][slice][col], slice-major so a whole tk=4 quad", - "// (64 fp32) is contiguous -> one LMUL=4 vle32.v loads it into 4 vregs.", - "// quad ptr for (m_tile mt, k_group g) = Apack_dram + (mt*(K/4)+g)*64.", - f'static float Apack_dram[{K}*{M}] __attribute__((section(".data"))) =', - array_to_cstr_f32(A_pack) + ";", - "", - "", - "// B_pack[n_tile][k_group][slice][col]; quad ptr for (n_tile nt, k_group g)", - "// = Bpack_dram + (nt*(K/4)+g)*64.", - f'static float Bpack_dram[{K}*{N}] __attribute__((section(".data"))) =', - array_to_cstr_f32(B_pack) + ";", - "", - ] - - out_dir = pathlib.Path(__file__).parent.parent / "data" - out_dir.mkdir(parents=True, exist_ok=True) - out_file = out_dir / f"data_{M}_{N}_{K}.h" - - with out_file.open("w") as f: - f.write("\n".join(lines)) - - print(f"Written: {out_file}") - -if __name__ == "__main__": - main() diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/CMakeLists.txt b/sw/vme/apps/tew32/fp32-matmul-vtse/CMakeLists.txt index 7c32ba213..46c342f3e 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/CMakeLists.txt +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/CMakeLists.txt @@ -41,17 +41,18 @@ function(add_fp32_vtse_variant precision variant source_dir) set(_tile_edge 16) endif() - # Use double buffering only when both input panels fit in the - # 512-KiB TCDM. Larger panels use a single buffer and are refilled - # synchronously by main.c. + # Generate only variants whose two input panels fit in TCDM. math(EXPR _panel_bytes "32 * ${_tile_edge} * ${_k} + 64 * ${_tile_edge} * ${_tile_edge}") math(EXPR _double_buffer_bytes "2 * ${_panel_bytes}") - if(_double_buffer_bytes GREATER 524288) - set(_buffer_count 1) - else() - set(_buffer_count 2) + if(_double_buffer_bytes GREATER SNRT_TCDM_SIZE) + message(STATUS + "Skipping FP32 VTSE ${precision}/${variant} M${_m} N${_n} K${_k}: " + "double buffering requires ${_double_buffer_bytes} bytes, " + "but TCDM provides ${SNRT_TCDM_SIZE} bytes") + continue() endif() + set(_buffer_count 2) set(_variant_build_dir "${CMAKE_CURRENT_BINARY_DIR}/generated/${precision}/${variant}/K${_k}") @@ -84,21 +85,18 @@ function(add_fp32_vtse_variant precision variant source_dir) add_custom_target(generate-${_target} DEPENDS "${_header}") add_snitch_test(${_target} "${source_dir}/main.c") add_dependencies(${_test_target} generate-${_target}) - set_target_properties(${_test_target} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/vme/apps") target_include_directories(${_test_target} PRIVATE "${source_dir}" "${source_dir}/kernel" "${source_dir}/data") target_compile_definitions(${_test_target} PRIVATE - MATMUL_K_CHUNK=${_k_chunk} MATMUL_BUFFER_COUNT=${_buffer_count} DATAHEADER="${_data_header}") endif() endforeach() endfunction() -foreach(_precision IN ITEMS te16 te32) +foreach(_precision IN ITEMS ${VME_TILE_VARIANT}) if(_precision STREQUAL "te32") set(_precision_shapes ${_fp32_vtse_te32_shapes}) set(_unrolled_ks 64 128 256) diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c index 91847ae65..6b0e09c5a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -41,18 +38,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma - const uintptr_t tile_bytes = operand_bytes * K; + const uintptr_t tile_bytes = operand_bytes * K_CHUNK; const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( - "mv %[rows], %[row_blocks]\n" + "mv %[rows], %[ROW_BLOCKS]\n" ".Lrow_%=:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" ".Lcol_%=:\n" // K-Group 1 "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=2, twiden=1 @@ -454,8 +451,8 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [cols] "=&r"(col_counter), [loop] "=&r"(loop_counter) : - [row_blocks] "r"((uintptr_t)row_blocks), - [col_blocks] "r"((uintptr_t)col_blocks), + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [tile_bytes] "r"(tile_bytes), diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-afterK/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c index b1cdddc1f..0c5bba91d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.c @@ -20,29 +20,26 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma const uintptr_t rewind_b0_bytes = operand_bytes * TE; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -447,14 +444,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" @@ -606,6 +603,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/store-interleave/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c index 3962c622b..3e20171d8 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.c @@ -20,30 +20,27 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (2u << 5) | 1u; const uintptr_t matrix_vtype = 0xD0; // e32, m1, ta, ma const uintptr_t rewind_b0_bytes = operand_bytes * TE; - const uintptr_t middle_k_groups = (uintptr_t)K / 8 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 8 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -368,14 +365,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 4\n" "addi %[loop], %[loop], -128\n" "sub %[loop], x0, %[loop]\n" @@ -498,6 +495,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/rolled/vsetvl/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c index 4aff5fdc9..d09623bcf 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -5092,6 +5091,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K128/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c index c0bae6560..aac62f9c7 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -9763,6 +9762,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K256/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.c index afbcf5820..e4450e0ab 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_afterK/K64/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c index d1ee132e4..9e7a58ca8 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -3990,14 +3989,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f // "sub %[cols], %[b1p], %[b0p]\n" // "add %[a0p], %[a0p], %[cols]\n" // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" + // "slli %[loop], %[COL_BLOCKS], 1\n" // "addi %[loop], %[loop], -1\n" // "6:\n" // "sub %[b0p], %[b0p], %[cols]\n" // "sub %[b1p], %[b1p], %[cols]\n" // "addi %[loop], %[loop], -1\n" // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" + // "mv %[cols], %[COL_BLOCKS]\n" // "slli %[loop], %[c_stride], 4\n" // "addi %[loop], %[loop], -128\n" // "sub %[loop], x0, %[loop]\n" @@ -5363,6 +5362,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K128/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c index 364a0002e..1cec2c0e8 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -7686,14 +7685,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f // "sub %[cols], %[b1p], %[b0p]\n" // "add %[a0p], %[a0p], %[cols]\n" // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" + // "slli %[loop], %[COL_BLOCKS], 1\n" // "addi %[loop], %[loop], -1\n" // "6:\n" // "sub %[b0p], %[b0p], %[cols]\n" // "sub %[b1p], %[b1p], %[cols]\n" // "addi %[loop], %[loop], -1\n" // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" + // "mv %[cols], %[COL_BLOCKS]\n" // "slli %[loop], %[c_stride], 4\n" // "addi %[loop], %[loop], -128\n" // "sub %[loop], x0, %[loop]\n" @@ -10291,6 +10290,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K256/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.c index c22579347..2f716b686 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -2142,14 +2141,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f // "sub %[cols], %[b1p], %[b0p]\n" // "add %[a0p], %[a0p], %[cols]\n" // "add %[a1p], %[a1p], %[cols]\n" - // "slli %[loop], %[col_blocks], 1\n" + // "slli %[loop], %[COL_BLOCKS], 1\n" // "addi %[loop], %[loop], -1\n" // "6:\n" // "sub %[b0p], %[b0p], %[cols]\n" // "sub %[b1p], %[b1p], %[cols]\n" // "addi %[loop], %[loop], -1\n" // "bnez %[loop], 6b\n" - // "mv %[cols], %[col_blocks]\n" + // "mv %[cols], %[COL_BLOCKS]\n" // "slli %[loop], %[c_stride], 4\n" // "addi %[loop], %[loop], -128\n" // "sub %[loop], x0, %[loop]\n" @@ -2899,6 +2898,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/store_interleave/K64/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c index f45835b14..367229083 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -4837,6 +4836,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K128/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c index 7b2576317..ee8fe9efd 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -9507,6 +9506,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K256/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.c index 76434c31d..ae66edf4d 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.c @@ -20,13 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -2501,6 +2500,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype) // [rewind_b0_bytes] "r"(rewind_b0_bytes), // [middle_k_groups] "r"(middle_k_groups), - // [col_blocks] "r"((uintptr_t)col_blocks) + // [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.h index 3f7dcd44e..29981b41a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 16, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/script/gen_data.py index e14f4ecdb..ea2c9faf3 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te16/unrolled/vsetvl/K64/script/gen_data.py @@ -76,10 +76,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c index 0be7ade84..acdacf454 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; uintptr_t c00 = (uintptr_t)(C); @@ -41,18 +38,18 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (1u << 5) | 1u; const uintptr_t matrix_vtype = 0xD1; // e32, m2, ta, ma - const uintptr_t tile_bytes = operand_bytes * K; + const uintptr_t tile_bytes = operand_bytes * K_CHUNK; const uintptr_t store_rows_bytes = TE * c_stride; const uintptr_t col_c_back = store_rows_bytes - 2 * operand_bytes; const uintptr_t row_c_advance = - store_rows_bytes - (col_blocks - 1) * 2 * operand_bytes; - const uintptr_t row_b_rewind = (2 * col_blocks - 1) * tile_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 4 - 3; + store_rows_bytes - (COL_BLOCKS - 1) * 2 * operand_bytes; + const uintptr_t row_b_rewind = (2 * COL_BLOCKS - 1) * tile_bytes; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( - "mv %[rows], %[row_blocks]\n" + "mv %[rows], %[ROW_BLOCKS]\n" ".Lrow_%=:\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" ".Lcol_%=:\n" // K-Group 1 "msetmtype %[mtype], %[vtype]\n" // set sew=32, set tm=TE, tk=1, twiden=1 @@ -318,8 +315,8 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [cols] "=&r"(col_counter), [loop] "=&r"(loop_counter) : - [row_blocks] "r"((uintptr_t)row_blocks), - [col_blocks] "r"((uintptr_t)col_blocks), + [ROW_BLOCKS] "r"((uintptr_t)ROW_BLOCKS), + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS), [c_stride] "r"(c_stride), [operand_bytes] "r"(operand_bytes), [tile_bytes] "r"(tile_bytes), diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h index 94b72e13b..9a4231641 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py index 13fb92600..ed06ea6c0 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-afterK/script/gen_data.py @@ -72,10 +72,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c index aa6076a29..536da3b9b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; const uintptr_t c_block_bytes = 2 * operand_bytes; @@ -36,14 +33,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (1u << 5) | 1u; const uintptr_t matrix_vtype = 0xD1; // e32, m2, ta, ma const uintptr_t rewind_b0_bytes = 8 * operand_bytes; - const uintptr_t middle_k_groups = (uintptr_t)K / 4 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -334,14 +331,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 5\n" "sub %[loop], %[loop], %[c_block_bytes]\n" "sub %[loop], x0, %[loop]\n" @@ -457,6 +454,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h index 94b72e13b..9a4231641 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py index 13fb92600..ed06ea6c0 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/store-interleave/script/gen_data.py @@ -72,10 +72,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c index 9992460bf..d39601f5a 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.c @@ -20,15 +20,12 @@ #include "matmul.h" -__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K) +__attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const float *B, float *C) { - const uint32_t col_blocks = (N + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t row_blocks = (M + BLOCK_DIM - 1) / BLOCK_DIM; - const uint32_t tile_stride = TE * K; uintptr_t a0p = (uintptr_t)At; - uintptr_t a1p = (uintptr_t)(At + tile_stride); + uintptr_t a1p = (uintptr_t)(At + TILE_STRIDE); uintptr_t b0p = (uintptr_t)B; - uintptr_t b1p = (uintptr_t)(B + tile_stride); + uintptr_t b1p = (uintptr_t)(B + TILE_STRIDE); uintptr_t tss; const uintptr_t operand_bytes = sizeof(float) * TE; const uintptr_t operand_bytes_m2 = operand_bytes; @@ -38,15 +35,15 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f uintptr_t c01 = c00 + operand_bytes; uintptr_t c10 = (uintptr_t)(C + TE * CHUNK_SIZE); uintptr_t c11 = (uintptr_t)(C + TE * CHUNK_SIZE + TE); - uintptr_t block_counter = (uintptr_t)row_blocks * col_blocks; - uintptr_t col_counter = (uintptr_t)col_blocks; + uintptr_t block_counter = (uintptr_t)ROW_BLOCKS * COL_BLOCKS; + uintptr_t col_counter = (uintptr_t)COL_BLOCKS; uintptr_t loop_counter; uintptr_t vl_counter; const uintptr_t c_stride = CHUNK_SIZE * sizeof(float); const uintptr_t matrix_mtype = (TE << 10) | (1u << 5) | 1u; const uintptr_t matrix_vtype = 0xD1; // e32, m2, ta, ma const uintptr_t rewind_b0_bytes = 2 * operand_bytes_m8; - const uintptr_t middle_k_groups = (uintptr_t)K / 4 - 3; + const uintptr_t middle_k_groups = (uintptr_t)K_CHUNK / 4 - 3; asm volatile( ".Lmatmul_block_%=:\n" @@ -312,14 +309,14 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f "sub %[cols], %[b1p], %[b0p]\n" "add %[a0p], %[a0p], %[cols]\n" "add %[a1p], %[a1p], %[cols]\n" - "slli %[loop], %[col_blocks], 1\n" + "slli %[loop], %[COL_BLOCKS], 1\n" "addi %[loop], %[loop], -1\n" "6:\n" "sub %[b0p], %[b0p], %[cols]\n" "sub %[b1p], %[b1p], %[cols]\n" "addi %[loop], %[loop], -1\n" "bnez %[loop], 6b\n" - "mv %[cols], %[col_blocks]\n" + "mv %[cols], %[COL_BLOCKS]\n" "slli %[loop], %[c_stride], 5\n" "sub %[loop], %[loop], %[c_block_bytes]\n" "sub %[loop], x0, %[loop]\n" @@ -436,6 +433,6 @@ __attribute__((noinline, aligned(64))) void matmul_fp32(const float *At, const f [vtype] "r"(matrix_vtype), [rewind_b0_bytes] "r"(rewind_b0_bytes), [middle_k_groups] "r"(middle_k_groups), - [col_blocks] "r"((uintptr_t)col_blocks) + [COL_BLOCKS] "r"((uintptr_t)COL_BLOCKS) : "memory"); } diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h index 94b72e13b..9a4231641 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/kernel/matmul.h @@ -25,6 +25,27 @@ enum { TE = 32, CHUNK_SIZE = 4 * TE, BLOCK_DIM = 2 * TE, + ROW_BLOCKS = (M_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + COL_BLOCKS = (N_CHUNK + BLOCK_DIM - 1) / BLOCK_DIM, + TILE_STRIDE = TE * K_CHUNK, }; -void matmul_fp32(const float *At, const float *B, float *C, uint32_t M, uint32_t N, uint32_t K); +enum { + TILE_INPUTS = TE * K_CHUNK, + TILE_BYTES = TILE_INPUTS * sizeof(float), + A_TILES = M_CHUNK / TE, + B_TILES = N_CHUNK / TE, + A_BYTES = A_TILES * TILE_BYTES, + B_BYTES = B_TILES * TILE_BYTES, + C_ELEMENTS = M_CHUNK * N_CHUNK, + C_BYTES = C_ELEMENTS * sizeof(float), + A_OFFSET = 0, + B_OFFSET = A_OFFSET + A_BYTES, + C_OFFSET = B_OFFSET + B_BYTES, + BUFFER_BYTES = C_OFFSET + C_BYTES, + L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, + L1_BUFFER_STRIDE = SNRT_TCDM_SIZE / L1_BUFFER_COUNT, + L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, +}; + +void matmul_fp32(const float *At, const float *B, float *C); diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/main.c b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/main.c index e70430731..82a425b7b 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/main.c +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/main.c @@ -19,43 +19,20 @@ #include #include -#include "matmul.h" #include DATAHEADER +#include "matmul.h" #include "kernel/matmul.c" -static inline uint32_t get_cycle(void) -{ - uint32_t c; - asm volatile("csrr %0, mcycle" : "=r"(c)); - return c; +static inline uint32_t get_cycle(void) { + uint32_t cycle; + asm volatile("csrr %0, mcycle" : "=r"(cycle)); + return cycle; } -enum { - TCDM_BYTES = 512 * 1024, - M_CHUNK = CHUNK_SIZE, - N_CHUNK = CHUNK_SIZE, - K_CHUNK = MATMUL_K_CHUNK, - TILE_INPUTS = TE * K_CHUNK, - TILE_BYTES = TILE_INPUTS * sizeof(float), - A_TILES = M_CHUNK / TE, - B_TILES = N_CHUNK / TE, - A_BYTES = A_TILES * TILE_BYTES, - B_BYTES = B_TILES * TILE_BYTES, - C_ELEMENTS = M_CHUNK * N_CHUNK, - C_BYTES = C_ELEMENTS * sizeof(float), - A_OFFSET = 0, - B_OFFSET = A_OFFSET + A_BYTES, - C_OFFSET = B_OFFSET + B_BYTES, - BUFFER_BYTES = C_OFFSET + C_BYTES, - L1_BUFFER_COUNT = MATMUL_BUFFER_COUNT, - L1_BUFFER_STRIDE = TCDM_BYTES / L1_BUFFER_COUNT, - L1_BYTES = (L1_BUFFER_COUNT - 1) * L1_BUFFER_STRIDE + BUFFER_BYTES, -}; - _Static_assert(BUFFER_BYTES <= L1_BUFFER_STRIDE, "matmul panel exceeds its L1 buffer partition"); -_Static_assert(L1_BYTES <= TCDM_BYTES, - "matmul buffers exceed the 512 KiB TCDM"); +_Static_assert(L1_BYTES <= SNRT_TCDM_SIZE, + "matmul buffers exceed the configured TCDM capacity"); static const float zero_tile_dram[TILE_INPUTS] __attribute__((section(".dram"), aligned(64))) = {0}; @@ -78,15 +55,12 @@ static inline uint32_t float_bits(float value) { return bits.u; } -static snrt_dma_txid_t fill_panel(float *dst, const float *src, - uint32_t panel_tiles, - uint32_t valid_tiles, - uint32_t valid_k, - uint32_t src_tile_k) { +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_panel( + float *dst, const float *src, uint32_t panel_tiles, uint32_t valid_tiles, + uint32_t valid_k, uint32_t src_tile_k) { if (valid_tiles == panel_tiles && valid_k == K_CHUNK && - src_tile_k == K_CHUNK) { + src_tile_k == K_CHUNK) return snrt_dma_start_1d(dst, src, panel_tiles * TILE_BYTES); - } const snrt_dma_txid_t clear_tid = snrt_dma_start_2d( dst, zero_tile_dram, TILE_BYTES, TILE_BYTES, 0, panel_tiles); @@ -99,45 +73,37 @@ static snrt_dma_txid_t fill_panel(float *dst, const float *src, valid_tiles); } -static snrt_dma_txid_t fill_a(float *dst, uint32_t mb, uint32_t kb) { - const uint32_t first_tile = mb * A_TILES; - const uint32_t tile_count = - (matmul_l.M + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(A_TILES, - tile_count - first_tile) - : 0; - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); - const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Apack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, A_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_b(float *dst, uint32_t nb, uint32_t kb) { - const uint32_t first_tile = nb * B_TILES; - const uint32_t tile_count = - (matmul_l.N + TE - 1) / TE; - const uint32_t valid_tiles = first_tile < tile_count - ? min_u32(B_TILES, - tile_count - first_tile) - : 0; +static inline __attribute__((always_inline)) snrt_dma_txid_t fill_buffer( + uint8_t *l1_base, uint32_t buffer, uint32_t mb, uint32_t nb, + uint32_t kb) { + uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; + const uint32_t a_first_tile = mb * A_TILES; + const uint32_t b_first_tile = nb * B_TILES; + const uint32_t a_tile_count = (matmul_l.M + TE - 1) / TE; + const uint32_t b_tile_count = (matmul_l.N + TE - 1) / TE; + const uint32_t a_valid_tiles = + a_first_tile < a_tile_count + ? min_u32(A_TILES, a_tile_count - a_first_tile) + : 0; + const uint32_t b_valid_tiles = + b_first_tile < b_tile_count + ? min_u32(B_TILES, b_tile_count - b_first_tile) + : 0; const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); const uint32_t k0 = kb * K_CHUNK; - const float *src = valid_tiles == 0 - ? zero_tile_dram - : Bpack_dram + - (first_tile * matmul_l.K + k0) * TE; - return fill_panel(dst, src, B_TILES, valid_tiles, valid_k, matmul_l.K); -} - -static snrt_dma_txid_t fill_buffer(uint8_t *l1_base, uint32_t buffer, - uint32_t mb, uint32_t nb, uint32_t kb) { - uint8_t *base = l1_base + buffer * L1_BUFFER_STRIDE; - fill_a((float *)(base + A_OFFSET), mb, kb); - return fill_b((float *)(base + B_OFFSET), nb, kb); + const float *a_src = + a_valid_tiles == 0 + ? zero_tile_dram + : Apack_dram + (a_first_tile * (matmul_l.K) + k0) * TE; + const float *b_src = + b_valid_tiles == 0 + ? zero_tile_dram + : Bpack_dram + (b_first_tile * (matmul_l.K) + k0) * TE; + + fill_panel((float *)(base + A_OFFSET), a_src, A_TILES, + a_valid_tiles, valid_k, matmul_l.K); + return fill_panel((float *)(base + B_OFFSET), b_src, B_TILES, + b_valid_tiles, valid_k, matmul_l.K); } int main(void) { @@ -172,18 +138,11 @@ int main(void) { uint32_t start_cycle = 0; uint32_t end_cycle = 0; for (uint32_t iter = 0; iter < iterations; ++iter) { - const uint32_t job = iter / kb_count; - const uint32_t mb = job / nb_count; - const uint32_t nb = job % nb_count; - const uint32_t kb = iter % kb_count; - const uint32_t valid_m = chunk_extent(matmul_l.M, mb, M_CHUNK); - const uint32_t valid_n = chunk_extent(matmul_l.N, nb, N_CHUNK); - const uint32_t valid_k = chunk_extent(matmul_l.K, kb, K_CHUNK); uint8_t *base = l1_base + current_buffer * L1_BUFFER_STRIDE; if (iter == 0) start_cycle = get_cycle(); - matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET), valid_m, valid_n, valid_k); + matmul_fp32((const float *)(base + A_OFFSET), (const float *)(base + B_OFFSET), (float *)(base + C_OFFSET)); end_cycle = get_cycle(); const snrt_dma_txid_t output_tid = snrt_dma_start_1d( diff --git a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py index 13fb92600..ed06ea6c0 100644 --- a/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py +++ b/sw/vme/apps/tew32/fp32-matmul-vtse/te32/rolled/vsetvl/script/gen_data.py @@ -72,10 +72,16 @@ def main(): "", '#include "layer.h"', "", + "enum {", + f" M_CHUNK = {M},", + f" N_CHUNK = {N},", + f" K_CHUNK = {K},", + "};", + "", "const matmul_layer matmul_l = {", - f" .M = {M},", - f" .N = {N},", - f" .K = {K},", + " .M = M_CHUNK,", + " .N = N_CHUNK,", + " .K = K_CHUNK,", "};", "", "// C[M*N] row-major fp32 golden reference, C = A*B", diff --git a/sw/vme/isaTests/CMakeLists.txt b/sw/vme/isaTests/CMakeLists.txt index 718e639ab..343ea36dd 100644 --- a/sw/vme/isaTests/CMakeLists.txt +++ b/sw/vme/isaTests/CMakeLists.txt @@ -1,8 +1,8 @@ -# Copyright 2024 ETH Zurich and University of Bologna. +# Copyright 2026 ETH Zurich and University of Bologna. # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 # -# Author: Pei-Yu Lin +# Author: Pei-Yu Lin, ETH Zürich cmake_minimum_required(VERSION 3.13) @@ -14,9 +14,9 @@ add_snitch_test(msetmtypei msetmtypei.c) add_snitch_test(msettn msettn.c) add_snitch_test(msettm msettm.c) add_snitch_test(msettk msettk.c) -add_snitch_test(vtmm vtmm.c) +# add_snitch_test(vtmm vtmm.c) # Integer matrix MAC is not implemented. add_snitch_test(vtfmm vtfmm.c) add_snitch_test(vtle vtle.c) add_snitch_test(vtse vtse.c) add_snitch_test(vtmv vtmv.c) -add_snitch_test(vtzero vtzero.c) \ No newline at end of file +add_snitch_test(vtzero vtzero.c) diff --git a/sw/vme/isaTests/msetmtype.c b/sw/vme/isaTests/msetmtype.c index 1a7064f89..2fbc5dcff 100644 --- a/sw/vme/isaTests/msetmtype.c +++ b/sw/vme/isaTests/msetmtype.c @@ -2,186 +2,60 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// msetmtype — Register matrix-type configuration (analogous to vsetvl) -// -// ------------------------------------------------------- // Test cases: -// TC1: mtype=0x03 (mtwiden=3), vsew=e8 -// TC2: mtype=0x02 (mtwiden=2), vsew=e16 -// TC3: mtype=0x01 (mtwiden=1), vsew=e32 -// TC4: mtype=0x00 (mtwiden=0, no widening) -// TC5: Overwrite — second write must override first -// TC6: vl always reset to 0 after msetmtype -// TC7: msetmtype vs msetmtypei equivalence +// TC1: apply TM, TK, MTWIDEN, and altfmt from register operands. +// TC2: clamp a register-supplied TK to KMAX while preserving TM. +// TC3: preserve the vector configuration when matrix mode is disabled. #include #include "vector_macros.h" #include "vector_matrix_macros.h" +// Restore nonzero TM/TK from rs1 and altfmt from rs2. void TEST_CASE1(void) { - uintptr_t mtype_val, vtype_val, vl_val; - uintptr_t new_mtype = 3u; // mtwiden=3 - uintptr_t new_vtype = 0xC0u; // vsew=e8 + uintptr_t mtype_val, vtype_val; + uintptr_t new_mtype = MTYPE(2, KMAX(16, 2), 4); + uintptr_t new_vtype = VTYPE(1, 0, 0, 0, 1); asm volatile("msetmtype %0, %1" :: "r"(new_mtype), "r"(new_vtype) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - + asm volatile("csrr %0, 0xC23" : "=r"(mtype_val)); + asm volatile("csrr %0, vtype" : "=r"(vtype_val)); XCMP(1, mtype_val, new_mtype); - XCMP(2, vtype_val, new_vtype); - XCMP(3, vl_val, 0); + XCMP(2, (vtype_val & ~(uintptr_t)7), VTYPE(1, 0, 1, 1, 1)); } +// Clamp register-supplied TK to KMAX while retaining a valid TM. void TEST_CASE2(void) { - uintptr_t mtype_val, vtype_val, vl_val; - uintptr_t new_mtype = 2u; // mtwiden=2 - uintptr_t new_vtype = 0xC8u; // vsew=e16 + uintptr_t mtype_val; + uintptr_t new_mtype = MTYPE(1, 7, 4); + uintptr_t new_vtype = VTYPE(2, 0, 0, 0, 0); asm volatile("msetmtype %0, %1" :: "r"(new_mtype), "r"(new_vtype) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(4, mtype_val, new_mtype); - XCMP(5, vtype_val, new_vtype); - XCMP(6, vl_val, 0); + asm volatile("csrr %0, 0xC23" : "=r"(mtype_val)); + XCMP(3, MTYPE_TK(mtype_val), KMAX(32, 1)); + XCMP(4, MTYPE_TM(mtype_val), 4); + XCMP(5, MTYPE_MTWIDEN(mtype_val), 1); } +// With matrix mode disabled, retain rs2's vector LMUL and tail/mask policy. void TEST_CASE3(void) { - uintptr_t mtype_val, vtype_val, vl_val; - uintptr_t new_mtype = 1u; // mtwiden=1 - uintptr_t new_vtype = 0xD0u; // vsew=e32 - - asm volatile("msetmtype %0, %1" :: "r"(new_mtype), "r"(new_vtype) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(7, mtype_val, new_mtype); - XCMP(8, vtype_val, new_vtype); - XCMP(9, vl_val, 0); -} - -void TEST_CASE4(void) { - uintptr_t mtype_val, vtype_val, vl_val; - uintptr_t new_mtype = 0u; // no widening - uintptr_t new_vtype = 0xC0u; // vsew=e8 - - asm volatile("msetmtype %0, %1" :: "r"(new_mtype), "r"(new_vtype) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(10, mtype_val, new_mtype); - XCMP(11, vtype_val, new_vtype); - XCMP(12, vl_val, 0); -} - -// ------------------------------------------------------- -// TC5: Overwrite — second write must override first completely -// CSR must reflect SECOND write only -// ------------------------------------------------------- -void TEST_CASE5(void) { uintptr_t mtype_val, vtype_val; + uintptr_t new_mtype = MTYPE(0, 0, 0); + uintptr_t new_vtype = VTYPE(2, 1, 0, 0, 0); // e32, m2, tu, mu - uintptr_t mtype_a = 2u; // mtwiden=2 - uintptr_t vtype_a = 0xC8u; // vsew=e16 - asm volatile("msetmtype %0, %1" :: "r"(mtype_a), "r"(vtype_a) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - XCMP(13, mtype_val, mtype_a); - - uintptr_t mtype_b = 3u; // mtwiden=3 - uintptr_t vtype_b = 0xC0u; // e8 - asm volatile("msetmtype %0, %1" :: "r"(mtype_b), "r"(vtype_b) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); - - XCMP(14, mtype_val, mtype_b); - XCMP(15, vtype_val, vtype_b); -} - -// ------------------------------------------------------- -// TC6: vl always reset to 0 after msetmtype -// Set vl to non-zero via vsetvli, then msetmtype, vl must become 0. -// ------------------------------------------------------- -void TEST_CASE6(void) { - uintptr_t vl_val; - uintptr_t avl = 8; - - asm volatile("vsetvli x0, %[avl], e8, m1, ta, ma" :: [avl] "r"(avl) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - XCMP(16, vl_val, 8); - - uintptr_t new_mtype = 3u; - uintptr_t new_vtype = 0xC0u; asm volatile("msetmtype %0, %1" :: "r"(new_mtype), "r"(new_vtype) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - XCMP(17, vl_val, 0); -} - -// ------------------------------------------------------- -// TC7: msetmtype vs msetmtypei equivalence -// Both must produce identical mtype and vtype CSR values. -// Note: msetmtypei builds vtype internally; -// msetmtype uses the rs1 value directly — -// so rs1 must match what msetmtypei would compute. -// ------------------------------------------------------- -void TEST_CASE7(void) { - uintptr_t mtype_imm, vtype_imm; - uintptr_t mtype_reg, vtype_reg; - - // --- (mtwiden=3, vsew=e8) --- - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_imm)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_imm)); - uintptr_t m3 = 3u; - uintptr_t v0 = 0xC0u; - asm volatile("msetmtype %[rs2], %[rs1]" :: [rs2] "r"(m3), [rs1] "r"(v0) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_reg)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_reg)); - XCMP(18, mtype_reg, mtype_imm); - XCMP(19, vtype_reg, vtype_imm); - - // --- (mtwiden=2, vsew=e16) --- - asm volatile("msetmtypei 0x2, 0x1" ::: "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_imm)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_imm)); - uintptr_t m2 = 2u; - uintptr_t v1 = 0xC8u; - asm volatile("msetmtype %[rs2], %[rs1]" :: [rs2] "r"(m2), [rs1] "r"(v1) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_reg)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_reg)); - XCMP(20, mtype_reg, mtype_imm); - XCMP(21, vtype_reg, vtype_imm); - - // --- (mtwiden=1, vsew=e32) --- - asm volatile("msetmtypei 0x1, 0x2" ::: "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_imm)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_imm)); - uintptr_t m1 = 1u; - uintptr_t v2 = 0xD0u; - asm volatile("msetmtype %[rs2], %[rs1]" :: [rs2] "r"(m1), [rs1] "r"(v2) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_reg)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_reg)); - XCMP(22, mtype_reg, mtype_imm); - XCMP(23, vtype_reg, vtype_imm); + asm volatile("csrr %0, 0xC23" : "=r"(mtype_val)); + asm volatile("csrr %0, vtype" : "=r"(vtype_val)); + XCMP(6, mtype_val, 0); + XCMP(7, vtype_val, new_vtype); } int main(void) { INIT_CHECK(); - // enable_vec(); - // enable_vme(); - TEST_CASE1(); TEST_CASE2(); TEST_CASE3(); - TEST_CASE4(); - TEST_CASE5(); - TEST_CASE6(); - TEST_CASE7(); - EXIT_CHECK(); } diff --git a/sw/vme/isaTests/msetmtypei.c b/sw/vme/isaTests/msetmtypei.c index bf07ff224..7c546fc16 100644 --- a/sw/vme/isaTests/msetmtypei.c +++ b/sw/vme/isaTests/msetmtypei.c @@ -2,18 +2,12 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// msetmtypei — Immediate matrix-type configuration -// -// ------------------------------------------------------- // Test cases: -// TC1: msetmtypei 0x3, 0x0 → mtwiden=3, vsew=e8 -// TC2: msetmtypei 0x2, 0x1 → mtwiden=2, vsew=e16 -// TC3: msetmtypei 0x1, 0x2 → mtwiden=1, vsew=e32 -// TC4: msetmtypei 0x0, 0x0 → mtwiden=0, vsew=e8 (no widening) -// TC5: Overwrite — write twice, CSR reflects last write only -// TC6: vl is reset to 0 after msetmtypei +// TC1-TC4: configure each supported MTWIDEN/VSEW combination and disable it. +// TC5: replace prior TM, TK, and altfmt state with immediate configuration. +// TC6: reset vl to zero after immediate configuration. #include #include "vector_macros.h" @@ -22,7 +16,7 @@ void TEST_CASE1(void) { uintptr_t mtype_val, vtype_val, vl_val; - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); + asm volatile("msetmtypei 0x3, 0x0, 0" ::: "memory"); asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); @@ -35,7 +29,7 @@ void TEST_CASE1(void) { void TEST_CASE2(void) { uintptr_t mtype_val, vtype_val, vl_val; - asm volatile("msetmtypei 0x2, 0x1" ::: "memory"); + asm volatile("msetmtypei 0x2, 0x1, 0" ::: "memory"); asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); @@ -48,7 +42,7 @@ void TEST_CASE2(void) { void TEST_CASE3(void) { uintptr_t mtype_val, vtype_val, vl_val; - asm volatile("msetmtypei 0x1, 0x2" ::: "memory"); + asm volatile("msetmtypei 0x1, 0x2, 0" ::: "memory"); asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); @@ -61,7 +55,7 @@ void TEST_CASE3(void) { void TEST_CASE4(void) { uintptr_t mtype_val, vtype_val, vl_val; - asm volatile("msetmtypei 0x0, 0x0" ::: "memory"); + asm volatile("msetmtypei 0x0, 0x0, 0" ::: "memory"); asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); @@ -71,24 +65,25 @@ void TEST_CASE4(void) { XCMP(12, vl_val, 0); } -// ------------------------------------------------------- -// TC5: Overwrite -// First write: mtwiden=2, vsew=e16 → mtype=0x02 -// Second write: mtwiden=3, vsew=e8 → mtype=0x03 -// CSR must reflect LAST write only -// ------------------------------------------------------- +// Immediate configuration clears previous TM/TK and replaces altfmt. void TEST_CASE5(void) { uintptr_t mtype_val, vtype_val; - - asm volatile("msetmtypei 0x2, 0x1" ::: "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - XCMP(13, mtype_val, 2u); - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vtype" : [BUF]"=r"(vtype_val)); - XCMP(14, mtype_val, 3u); - XCMP(15, vtype_val, 0xC0u); + uintptr_t n = 4, k = KMAX(16, 2); + + asm volatile("msetmtypei 2, 1, 1" ::: "memory"); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); + asm volatile("msettk zero, %0" :: "r"(k) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype_val)); + asm volatile("csrr %0, vtype" : "=r"(vtype_val)); + XCMP(13, MTYPE_TM(mtype_val), n); + XCMP(14, MTYPE_TK(mtype_val), k); + XCMP(15, ((vtype_val >> 8) & 1), 1); + + asm volatile("msetmtypei 3, 0, 0" ::: "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype_val)); + asm volatile("csrr %0, vtype" : "=r"(vtype_val)); + XCMP(16, mtype_val, 3u); + XCMP(17, ((vtype_val >> 8) & 1), 0); } // ------------------------------------------------------- @@ -102,17 +97,15 @@ void TEST_CASE6(void) { asm volatile("vsetvli x0, %[avl], e8, m1, ta, ma" :: [avl] "r"(avl) : "memory"); asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - XCMP(16, vl_val, 8); + XCMP(18, vl_val, 8); - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); + asm volatile("msetmtypei 0x3, 0x0, 0" ::: "memory"); asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - XCMP(17, vl_val, 0); + XCMP(19, vl_val, 0); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); TEST_CASE1(); TEST_CASE2(); diff --git a/sw/vme/isaTests/msettk.c b/sw/vme/isaTests/msettk.c index e20fd697e..14630ea31 100644 --- a/sw/vme/isaTests/msettk.c +++ b/sw/vme/isaTests/msettk.c @@ -2,125 +2,68 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// msettk — Set tile K dimension (tk = inner loop / reduction dimension) -// -// ------------------------------------------------------- // Test cases: -// TC1: mtwiden=3, request tk=4 → expect min(4, KMAX=4) = 4 -// TC2: mtwiden=3, request tk > KMAX → expect clamped to 4 -// TC3: mtwiden=3, request tk=0 → expect 0 -// TC4: mtwiden=3, request tk=1 → expect 1 -// TC5: mtwiden=0 → new_tk forced to 0 -// TC6: returned rd value matches mtype[7:5] -// TC7: msettk does NOT change vl or tm +// For every supported SEW/TWIDEN pair, check TK requests below, at, and above +// KMAX, including a zero source and preserved matrix/vector state. #include #include "vector_macros.h" +#define VME_DIMENSION_TEST #include "vector_matrix_macros.h" -void TEST_CASE1(void) { - uintptr_t mtype_val; - uintptr_t req_tk = 4; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettk x0, %[tk]" :: [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(1, MTYPE_TK(mtype_val), 4); -} - -void TEST_CASE2(void) { - uintptr_t mtype_val; - uintptr_t req_tk = 100; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettk x0, %[tk]" :: [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(2, MTYPE_TK(mtype_val), KMAX); -} - -void TEST_CASE3(void) { - uintptr_t mtype_val; - uintptr_t req_tk = 0; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettk x0, %[tk]" :: [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(3, MTYPE_TK(mtype_val), 0); -} - -void TEST_CASE4(void) { - uintptr_t mtype_val; - uintptr_t req_tk = 1; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettk x0, %[tk]" :: [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(4, MTYPE_TK(mtype_val), 1); -} - -void TEST_CASE5(void) { - uintptr_t mtype_val; - uintptr_t req_tk = 4; - - asm volatile("msetmtypei 0x0, 0x0" ::: "memory"); - asm volatile("msettk x0, %[tk]" :: [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(5, MTYPE_TK(mtype_val), 0); -} - -void TEST_CASE6(void) { - uintptr_t rd_val = 0xDEAD; - uintptr_t mtype_val; - uintptr_t req_tk = 4; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettk %[rd], %[tk]" : [rd]"=r"(rd_val) : [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(6, rd_val, 4); - XCMP(7, MTYPE_TK(mtype_val), rd_val); -} - -void TEST_CASE7(void) { - uintptr_t vl_before, vl_after; - uintptr_t mtype_before, mtype_after; - uintptr_t req_tn = 4, req_tm = 4, req_tk = 4; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettn x0, %[tn]" :: [tn]"r"(req_tn) : "memory"); - asm volatile("msettm x0, %[tm]" :: [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_before)); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_before)); - - asm volatile("msettk x0, %[tk]" :: [tk]"r"(req_tk) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_after)); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_after)); - - XCMP(8, vl_after, vl_before); - XCMP(9, MTYPE_TM(mtype_after), MTYPE_TM(mtype_before)); - XCMP(10, MTYPE_MTWIDEN(mtype_after), MTYPE_MTWIDEN(mtype_before)); - XCMP(11, MTYPE_TK(mtype_after), KMAX); +// Each case checks one request independently of previous test state. +static void check_request(const vme_test_config_t *cfg, uintptr_t vlenb, + uintptr_t request, unsigned id, int zero_source) { + uintptr_t rd, mtype, vtype, vl; + uintptr_t expected = vme_test_min(request, cfg->kmax); + uintptr_t initial_mtype = MTYPE(cfg->mtwiden, 1, 1); + vme_test_setup(cfg); + asm volatile("csrr %0, vtype" : "=r"(vtype)); + XCMP(id, vtype, vme_test_vtype(cfg, vlenb)); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + XCMP(id + 1, mtype, initial_mtype); + + // Preserve a nonzero VL so an accidental clear is observable. + uintptr_t seed_vl = 1; + asm volatile("msettn zero, %0" :: "r"(seed_vl) : "memory"); + asm volatile("csrr %0, vl" : "=r"(vl)); + XCMP(id + 8, vl, seed_vl); + + if (zero_source) + asm volatile("msettk %0, zero" : "=r"(rd) :: "memory"); + else + asm volatile("msettk %0, %1" : "=r"(rd) : "r"(request) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + asm volatile("csrr %0, vl" : "=r"(vl)); + asm volatile("csrr %0, vtype" : "=r"(vtype)); + XCMP(id + 2, rd, expected); + XCMP(id + 3, MTYPE_TK(mtype), expected); + XCMP(id + 4, (mtype & ~((uintptr_t)7 << 5)), (initial_mtype & ~((uintptr_t)7 << 5))); + XCMP(id + 5, vtype, vme_test_vtype(cfg, vlenb)); + XCMP(id + 6, vl, seed_vl); + + // rd=x0 discards the result but must still update the destination CSR. + vme_test_setup(cfg); + asm volatile("msettk zero, %0" :: "r"(request) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + asm volatile("csrr %0, vl" : "=r"(vl)); + XCMP(id + 7, MTYPE_TK(mtype), expected); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); - - TEST_CASE1(); - TEST_CASE2(); - TEST_CASE3(); - TEST_CASE4(); - TEST_CASE5(); - TEST_CASE6(); - TEST_CASE7(); - + uintptr_t vlenb; + asm volatile("csrr %0, vlenb" : "=r"(vlenb)); + for (unsigned c = 0; c < sizeof(vme_test_configs) / sizeof(vme_test_configs[0]); ++c) { + const vme_test_config_t *cfg = &vme_test_configs[c]; + uintptr_t limit = cfg->kmax; + uintptr_t requests[] = {0, 1, limit - 1, limit, limit + 1, UINTPTR_MAX, 0}; + printf("msettk: SEW=%u TWIDEN=%u TE=%u capacity=%u\n", + cfg->sew, cfg->twiden, VME_TEST_TE, (unsigned)limit); + for (unsigned r = 0; r < sizeof(requests) / sizeof(requests[0]); ++r) + check_request(cfg, vlenb, requests[r], 100 * (c + 1) + 10 * r, r == 6); + } EXIT_CHECK(); } diff --git a/sw/vme/isaTests/msettm.c b/sw/vme/isaTests/msettm.c index a8c23cf5d..d464724f9 100644 --- a/sw/vme/isaTests/msettm.c +++ b/sw/vme/isaTests/msettm.c @@ -2,157 +2,68 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// msettm — Set tile M dimension (tm = number of output rows) +// Test cases: +// For every supported SEW/TWIDEN pair, check TM requests below, at, and above +// LMUL * EVE and TE, including a zero source and preserved matrix/vector state. #include #include "vector_macros.h" +#define VME_DIMENSION_TEST #include "vector_matrix_macros.h" -static inline uintptr_t min3(uintptr_t a, uintptr_t b, uintptr_t c) { - uintptr_t tmp; - tmp = (a < b) ? a : b; - return (tmp < c) ? tmp : c; -} - -void TEST_CASE1(void) { - uintptr_t rd_val, mtype_val; - uintptr_t req_tm = 4; - uintptr_t expected; - uintptr_t vlenb, vlen_bits, tew_bits; - uintptr_t lmul_eve, ete; - - asm volatile("csrr %0, vlenb" : "=r"(vlenb)); - vlen_bits = vlenb * 8u; - tew_bits = 8u * 4u; // SEW=e8, TWIDEN=4 => TEW=32 - lmul_eve = vlen_bits / tew_bits; - ete = (vlen_bits / 2u) / tew_bits; - expected = min3(req_tm, lmul_eve, ete); - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettm %[rd], %[tm]" : [rd]"=r"(rd_val) : [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(1, rd_val, expected); - XCMP(2, MTYPE_TM(mtype_val), expected); - XCMP(3, MTYPE_TM(mtype_val), rd_val); -} - -void TEST_CASE2(void) { - uintptr_t rd_val, mtype_val; - uintptr_t req_tm = (uintptr_t)-1; - uintptr_t expected; - uintptr_t vlenb, vlen_bits, tew_bits; - uintptr_t lmul_eve, ete; - - asm volatile("csrr %0, vlenb" : "=r"(vlenb)); - vlen_bits = vlenb * 8u; - tew_bits = 8u * 4u; // SEW=e8, TWIDEN=4 => TEW=32 - lmul_eve = vlen_bits / tew_bits; - ete = (vlen_bits / 2u) / tew_bits; - expected = min3(req_tm, lmul_eve, ete); - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettm %[rd], %[tm]" : [rd]"=r"(rd_val) : [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(4, rd_val, expected); - XCMP(5, MTYPE_TM(mtype_val), expected); - XCMP(6, MTYPE_TM(mtype_val), rd_val); -} - -void TEST_CASE3(void) { - uintptr_t rd_val, mtype_val; - uintptr_t req_tm = 0; - uintptr_t expected; - uintptr_t vlenb, vlen_bits, tew_bits; - uintptr_t lmul_eve, ete; - - asm volatile("csrr %0, vlenb" : "=r"(vlenb)); - vlen_bits = vlenb * 8u; - tew_bits = 8u * 4u; // SEW=e8, TWIDEN=4 => TEW=32 - lmul_eve = vlen_bits / tew_bits; - ete = (vlen_bits / 2u) / tew_bits; - expected = min3(req_tm, lmul_eve, ete); - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettm %[rd], %[tm]" : [rd]"=r"(rd_val) : [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(7, rd_val, expected); - XCMP(8, MTYPE_TM(mtype_val), expected); - XCMP(9, MTYPE_TM(mtype_val), rd_val); -} - -void TEST_CASE4(void) { - uintptr_t rd_val, mtype_val; - uintptr_t req_tm = 4; - uintptr_t expected = 0; - - asm volatile("msetmtypei 0x0, 0x0" ::: "memory"); - asm volatile("msettm %[rd], %[tm]" : [rd]"=r"(rd_val) : [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(10, rd_val, expected); - XCMP(11, MTYPE_TM(mtype_val), expected); - XCMP(12, MTYPE_TM(mtype_val), rd_val); -} - -void TEST_CASE5(void) { - uintptr_t rd_val, mtype_val; - uintptr_t req_tm = 4; - uintptr_t expected; - uintptr_t vlenb, vlen_bits, tew_bits; - uintptr_t lmul_eve, ete; - - asm volatile("csrr %0, vlenb" : "=r"(vlenb)); - vlen_bits = vlenb * 8u; - tew_bits = 8u * 4u; // SEW=e8, TWIDEN=4 => TEW=32 - lmul_eve = vlen_bits / tew_bits; - ete = (vlen_bits / 2u) / tew_bits; - expected = min3(req_tm, lmul_eve, ete); - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettm %[rd], %[tm]" : [rd]"=r"(rd_val) : [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - - XCMP(13, rd_val, expected); - XCMP(14, MTYPE_TM(mtype_val), expected); - XCMP(15, MTYPE_TM(mtype_val), rd_val); -} - -void TEST_CASE6(void) { - uintptr_t rd_val, mtype_val; - uintptr_t vl_before, vl_after; - uintptr_t req_tm = 4; - uintptr_t expected = 0; - - asm volatile("msetmtypei 0x0, 0x0" ::: "memory"); - asm volatile("vsetivli x0, 7, e8, m1, ta, ma" ::: "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_before)); - - asm volatile("msettm %[rd], %[tm]" : [rd]"=r"(rd_val) : [tm]"r"(req_tm) : "memory"); - asm volatile("csrr %[B], 0xC23" : [B]"=r"(mtype_val)); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_after)); - - XCMP(16, rd_val, expected); - XCMP(17, MTYPE_TM(mtype_val), expected); - XCMP(18, MTYPE_TM(mtype_val), rd_val); - XCMP(19, vl_after, vl_before); +// Each case checks one request independently of previous test state. +static void check_request(const vme_test_config_t *cfg, uintptr_t vlenb, + uintptr_t request, unsigned id, int zero_source) { + uintptr_t rd, mtype, vtype, vl; + uintptr_t expected = vme_test_min(request, vme_test_capacity(cfg, vlenb)); + uintptr_t initial_mtype = MTYPE(cfg->mtwiden, 1, 1); + vme_test_setup(cfg); + asm volatile("csrr %0, vtype" : "=r"(vtype)); + XCMP(id, vtype, vme_test_vtype(cfg, vlenb)); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + XCMP(id + 1, mtype, initial_mtype); + + // Preserve a nonzero VL so an accidental clear is observable. + uintptr_t seed_vl = 1; + asm volatile("msettn zero, %0" :: "r"(seed_vl) : "memory"); + asm volatile("csrr %0, vl" : "=r"(vl)); + XCMP(id + 8, vl, seed_vl); + + if (zero_source) + asm volatile("msettm %0, zero" : "=r"(rd) :: "memory"); + else + asm volatile("msettm %0, %1" : "=r"(rd) : "r"(request) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + asm volatile("csrr %0, vl" : "=r"(vl)); + asm volatile("csrr %0, vtype" : "=r"(vtype)); + XCMP(id + 2, rd, expected); + XCMP(id + 3, MTYPE_TM(mtype), expected); + XCMP(id + 4, (mtype & ~((uintptr_t)0x3FFF << 10)), (initial_mtype & ~((uintptr_t)0x3FFF << 10))); + XCMP(id + 5, vtype, vme_test_vtype(cfg, vlenb)); + XCMP(id + 6, vl, seed_vl); + + // rd=x0 discards the result but must still update the destination CSR. + vme_test_setup(cfg); + asm volatile("msettm zero, %0" :: "r"(request) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + asm volatile("csrr %0, vl" : "=r"(vl)); + XCMP(id + 7, MTYPE_TM(mtype), expected); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); - - TEST_CASE1(); - TEST_CASE2(); - TEST_CASE3(); - TEST_CASE4(); - TEST_CASE5(); - TEST_CASE6(); - + uintptr_t vlenb; + asm volatile("csrr %0, vlenb" : "=r"(vlenb)); + for (unsigned c = 0; c < sizeof(vme_test_configs) / sizeof(vme_test_configs[0]); ++c) { + const vme_test_config_t *cfg = &vme_test_configs[c]; + uintptr_t limit = vme_test_capacity(cfg, vlenb); + uintptr_t requests[] = {0, 1, limit - 1, limit, limit + 1, UINTPTR_MAX, 0}; + printf("msettm: SEW=%u TWIDEN=%u TE=%u capacity=%u\n", + cfg->sew, cfg->twiden, VME_TEST_TE, (unsigned)limit); + for (unsigned r = 0; r < sizeof(requests) / sizeof(requests[0]); ++r) + check_request(cfg, vlenb, requests[r], 100 * (c + 1) + 10 * r, r == 6); + } EXIT_CHECK(); } diff --git a/sw/vme/isaTests/msettn.c b/sw/vme/isaTests/msettn.c index 3cd78ce79..89170c5a4 100644 --- a/sw/vme/isaTests/msettn.c +++ b/sw/vme/isaTests/msettn.c @@ -2,105 +2,61 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// msettn — Set tile N dimension (tn = number of output columns) -// -// ------------------------------------------------------- // Test cases: -// TC1: mtwiden=3, SEW=e8, request tn=4 → min(4, ETE)=4 -// TC2: mtwiden=3, SEW=e8, request tn>ETE → clamped to ETE -// TC3: mtwiden=3, SEW=e8, request tn=0 → 0 -// TC4: mtwiden=0 — standard vl path, no ETE limit -// TC5: returned rd == vl CSR +// For every supported SEW/TWIDEN pair, check TN requests below, at, and above +// LMUL * EVE and TE, including a zero source and preserved matrix/vector state. #include #include "vector_macros.h" +#define VME_DIMENSION_TEST #include "vector_matrix_macros.h" -static inline uintptr_t min3(uintptr_t a, uintptr_t b, uintptr_t c) { - uintptr_t tmp; - tmp = (a < b) ? a : b; - return (tmp < c) ? tmp : c; -} - -void TEST_CASE1(void) { - uintptr_t vl_val; - uintptr_t req_tn = 4; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettn x0, %[tn]" :: [tn]"r"(req_tn) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(1, vl_val, 4); -} - -void TEST_CASE2(void) { - uintptr_t vl_val; - uintptr_t req_tn = 0xFFFF; - uintptr_t expected; - uintptr_t vlenb, vlen_bits, tew_bits; - uintptr_t lmul_eve, ete; - - asm volatile("csrr %0, vlenb" : "=r"(vlenb)); - vlen_bits = vlenb * 8u; - tew_bits = 8u * 4u; // SEW=e8, TWIDEN=4 => TEW=32 - lmul_eve = vlen_bits / tew_bits; - ete = (vlen_bits / 2u) / tew_bits; - expected = min3(req_tn, lmul_eve, ete); - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettn x0, %[tn]" :: [tn]"r"(req_tn) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(2, vl_val, expected); -} - -void TEST_CASE3(void) { - uintptr_t vl_val; - uintptr_t req_tn = 0; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettn x0, %[tn]" :: [tn]"r"(req_tn) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(3, vl_val, 0); -} - -void TEST_CASE4(void) { - uintptr_t vl_val; - uintptr_t req_tn = 8; - - asm volatile("msetmtypei 0x0, 0x0" ::: "memory"); - asm volatile("msettn x0, %[tn]" :: [tn]"r"(req_tn) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(4, vl_val, 8); -} - -void TEST_CASE5(void) { - uintptr_t rd_val = 0xDEAD; - uintptr_t vl_val; - uintptr_t req_tn = 4; - - asm volatile("msetmtypei 0x3, 0x0" ::: "memory"); - asm volatile("msettn %[rd], %[tn]" : [rd]"=r"(rd_val) : [tn]"r"(req_tn) : "memory"); - asm volatile("csrr %[BUF], vl" : [BUF]"=r"(vl_val)); - - XCMP(5, rd_val, 4); - XCMP(6, vl_val, rd_val); +// Each case checks one request independently of previous test state. +static void check_request(const vme_test_config_t *cfg, uintptr_t vlenb, + uintptr_t request, unsigned id, int zero_source) { + uintptr_t rd, mtype, vtype, vl; + uintptr_t expected = vme_test_min(request, vme_test_capacity(cfg, vlenb)); + uintptr_t initial_mtype = MTYPE(cfg->mtwiden, 1, 1); + vme_test_setup(cfg); + asm volatile("csrr %0, vtype" : "=r"(vtype)); + XCMP(id, vtype, vme_test_vtype(cfg, vlenb)); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + XCMP(id + 1, mtype, initial_mtype); + + if (zero_source) + asm volatile("msettn %0, zero" : "=r"(rd) :: "memory"); + else + asm volatile("msettn %0, %1" : "=r"(rd) : "r"(request) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + asm volatile("csrr %0, vl" : "=r"(vl)); + asm volatile("csrr %0, vtype" : "=r"(vtype)); + XCMP(id + 2, rd, expected); + XCMP(id + 3, vl, expected); + XCMP(id + 4, mtype, initial_mtype); + XCMP(id + 5, vtype, vme_test_vtype(cfg, vlenb)); + + // rd=x0 discards the result but must still update the destination CSR. + vme_test_setup(cfg); + asm volatile("msettn zero, %0" :: "r"(request) : "memory"); + asm volatile("csrr %0, 0xC23" : "=r"(mtype)); + asm volatile("csrr %0, vl" : "=r"(vl)); + XCMP(id + 7, vl, expected); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); - - TEST_CASE1(); - TEST_CASE2(); - TEST_CASE3(); - TEST_CASE4(); - TEST_CASE5(); - + uintptr_t vlenb; + asm volatile("csrr %0, vlenb" : "=r"(vlenb)); + for (unsigned c = 0; c < sizeof(vme_test_configs) / sizeof(vme_test_configs[0]); ++c) { + const vme_test_config_t *cfg = &vme_test_configs[c]; + uintptr_t limit = vme_test_capacity(cfg, vlenb); + uintptr_t requests[] = {0, 1, limit - 1, limit, limit + 1, UINTPTR_MAX, 0}; + printf("msettn: SEW=%u TWIDEN=%u TE=%u capacity=%u\n", + cfg->sew, cfg->twiden, VME_TEST_TE, (unsigned)limit); + for (unsigned r = 0; r < sizeof(requests) / sizeof(requests[0]); ++r) + check_request(cfg, vlenb, requests[r], 100 * (c + 1) + 10 * r, r == 6); + } EXIT_CHECK(); } diff --git a/sw/vme/isaTests/vtfmm.c b/sw/vme/isaTests/vtfmm.c index bbcc3f1fc..3903249c9 100644 --- a/sw/vme/isaTests/vtfmm.c +++ b/sw/vme/isaTests/vtfmm.c @@ -2,56 +2,36 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// vtfmm.c — ISA tests for vtfmm.tvv and vtfmm.alt.tvv -// -// ------------------------------------------------------- -// vtfmm.tvv: -// SEW=e16, TWIDEN=2 : fp16 * fp16 → fp32 acc (FP16→FP32 widening) -// SEW=e32, TWIDEN=1 : fp32 * fp32 → fp32 acc (direct, no widening) -// SEW=e8, TWIDEN=4 : E4M3 * E4M3 → fp32 acc (altfmt=0) -// SEW=e8, TWIDEN=4 : E4M3 * E5M2 → fp32 acc (altfmt=1) -// -// vtfmm.alt.tvv: -// SEW=e16, TWIDEN=4 : BF16 * BF16 → fp32 acc (BF16 inputs) -// SEW=e8, TWIDEN=4 : E5M2 * E4M3 → fp32 acc (altfmt=0) -// SEW=e8, TWIDEN=4 : E5M2 * E5M2 → fp32 acc (altfmt=1) -// -// ------------------------------------------------------- // Test cases: -// TC1 : vtfmm.tvv SEW=16, fp16→fp32, A*I=A -// TC2 : vtfmm.tvv SEW=16, fp16→fp32, known dot product 2×2 -// TC3 : vtfmm.tvv SEW=16, fp16→fp32, accumulation (×2) -// TC4 : vtfmm.alt SEW=16, BF16→fp32, A*I=A -// TC5 : vtfmm.alt SEW=16, BF16→fp32, known dot product 2×2 -// TC6 : vtfmm.alt SEW=16, BF16→fp32, accumulation (×2) -// TC7 : vtfmm.tvv SEW=32, fp32→fp32, known dot product -// TC8 : vtfmm.tvv SEW=32, fp32→fp32, accumulation (×2) -// TC9 : vtfmm.tvv SEW=8, E4M3*E4M3 (altfmt=0), TK=1 -// TC10: vtfmm.tvv SEW=8, E4M3*E5M2 (altfmt=1), TK=1 -// TC11: vtfmm.alt SEW=8, E5M2*E4M3 (altfmt=0), TK=1 -// TC12: vtfmm.alt SEW=16, BF16*BF16 altfmt isolation check -// -// Spatz vendor extension (mtype.vke, not part of the Zvt spec): lets a -// single vtfmm.tvv fuse tk>1 K-steps for SEW=32, beyond the spec's own -// KMAX=1 bound for that width. See spatz_pkg.sv.tpl's mtype_t.vke and -// spatz_controller.sv's kmax clamp relaxation. -// TC13: vtfmm.tvv SEW=32, vke=1, tk=2 fused — cross-checked against -// TC8's two-separate-dispatch accumulation (same inputs duplicated -// across both K-rows must give the identical result) -// TC14: vtfmm.tvv SEW=32, vke=1, tk=4 fused — four distinct K-rows, -// hand-computed expected result (catches beat-offset bugs that -// duplicated-data tests like TC13 can't) -// TC15: vtfmm.tvv SEW=32, vke=1 — sequencing/hazard check: fused tk=2 -// to mt0, then plain tk=1 to a different tile mt4, then plain -// tk=1 back to mt0; verifies no lost/double-counted contribution +// TC1-TC6: FP32 products, accumulation, and tile interleaving. +// TC7: FP32 full 8x8 tile accumulation (disabled). +// TC8-TC11: FP16-to-FP32 products, accumulation, and packed K lanes. +// TC12-TC15: BF16-to-FP32 products and altfmt selection (disabled). +// TC16-TC18: FP8-to-FP32 format combinations (disabled). #include #include "vector_macros.h" #include "float_macros.h" #include "vector_matrix_macros.h" +// Check every element so a failed tile reports its complete output row. +#define VTFMM_CMP_U32(casenum, vect, ...) do { \ + const uint32_t expected[] = {__VA_ARGS__}; \ + unsigned int mismatches = 0; \ + VSTORE_U32(vect); \ + for (unsigned int i = 0; i < sizeof(expected) / sizeof(expected[0]); ++i) { \ + if (Ru32[i] != expected[i]) { \ + printf("[TC %d] Index %u FAILED. Got 0x%08x, expected 0x%08x.\n", \ + (casenum), i, (unsigned)Ru32[i], (unsigned)expected[i]); \ + ++mismatches; \ + } \ + } \ + if (mismatches) ++num_failed; \ + else printf("[TC %d] PASSED.\n", (casenum)); \ +} while (0) + // ------------------------------------------------------- // FP8 constants // ------------------------------------------------------- @@ -84,87 +64,76 @@ #define BF16_7_0 0x40E0 // 7.0 (fp32 0x40E00000 >> 16) #define BF16_8_0 0x4100 // 8.0 (fp32 0x41000000 >> 16) -// TC1: vtfmm.tvv SEW=16, fp16→fp32, A*I=A — C[0]=[1.0,2.0] C[1]=[3.0,4.0] +// TC1: vtfmm.tvv SEW=32, fp32×fp32→fp32 — C[0]=[5.0,7.0] C[1]=[15.0,21.0] void TEST_CASE1(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; + const uintptr_t TN = 2, TM = 2, TK = KMAX(32, 1); - VSET(2, e16, m1); - VLOAD_16(v8, 0x3C00, 0x4200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 3.0] - VLOAD_16(v12, 0x4000, 0x4400, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [2.0, 4.0] - VLOAD_16(v16, 0x3C00, 0x0000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [1.0, 0.0] - VLOAD_16(v20, 0x0000, 0x3C00, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [0.0, 1.0] + VSET(2, e32, m1); + VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // [1.0, 3.0] + VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // [5.0, 7.0] - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - uintptr_t tss; tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(1, v1, 0x3F800000, 0x40000000); // row0: [1.0, 2.0] - VCMP_U32(2, v2, 0x40400000, 0x40800000); // row1: [3.0, 4.0] + VTFMM_CMP_U32(13, v1, 0x40A00000, 0x40E00000); // row0: [5.0, 7.0] + VTFMM_CMP_U32(14, v2, 0x41700000, 0x41A80000); // row1: [15.0, 21.0] asm volatile("vtdiscard" ::: "memory"); } -// TC2: vtfmm.tvv SEW=16, fp16→fp32, known dot product — C[0]=[17.0,23.0] C[1]=[39.0,53.0] +// TC2: vtfmm.tvv SEW=32, accumulation (×2) — C[0]=[10.0,14.0] C[1]=[30.0,42.0] void TEST_CASE2(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; + const uintptr_t TN = 2, TM = 2, TK = KMAX(32, 1); - VSET(2, e16, m1); - VLOAD_16(v8, 0x3C00, 0x4200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 3.0] - VLOAD_16(v12, 0x4000, 0x4400, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [2.0, 4.0] - VLOAD_16(v16, 0x4500, 0x4700, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [5.0, 7.0] - VLOAD_16(v20, 0x4600, 0x4800, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [6.0, 8.0] + VSET(2, e32, m1); + VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); uintptr_t tss; tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(3, v1, 0x41880000, 0x41B80000); // row0: [17.0, 23.0] - VCMP_U32(4, v2, 0x421C0000, 0x42540000); // row1: [39.0, 53.0] + VTFMM_CMP_U32(15, v1, 0x41200000, 0x41600000); // row0: [10.0, 14.0] + VTFMM_CMP_U32(16, v2, 0x41F00000, 0x42280000); // row1: [30.0, 42.0] asm volatile("vtdiscard" ::: "memory"); } -// TC3: vtfmm.tvv SEW=16, fp16→fp32, accumulation (×2) — C[0]=[2.0,4.0] C[1]=[6.0,8.0] +// TC3: FP32 accumulation with two explicit single-K instructions. void TEST_CASE3(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; + const uintptr_t TN = 2, TM = 2, TK = KMAX(32, 1); - VSET(2, e16, m1); - VLOAD_16(v8, 0x3C00, 0x4200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_16(v12, 0x4000, 0x4400, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_16(v16, 0x3C00, 0x0000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_16(v20, 0x0000, 0x3C00, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VSET(2, e32, m1); + VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 3.0] + VLOAD_32(v12, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [1.0, 3.0] + VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [5.0, 7.0] + VLOAD_32(v20, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [5.0, 7.0] - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v12, v20" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -172,172 +141,226 @@ void TEST_CASE3(void) { tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(5, v1, 0x40000000, 0x40800000); // row0: [2.0, 4.0] - VCMP_U32(6, v2, 0x40C00000, 0x41000000); // row1: [6.0, 8.0] + VTFMM_CMP_U32(25, v1, 0x41200000, 0x41600000); // row0: [10.0, 14.0] + VTFMM_CMP_U32(26, v2, 0x41F00000, 0x42280000); // row1: [30.0, 42.0] asm volatile("vtdiscard" ::: "memory"); } -// TC4: vtfmm.alt.tvv SEW=16, BF16→fp32, A*I=A — C[0]=[1.0,2.0] C[1]=[3.0,4.0] +// TC4: FP32 accumulation with four explicit single-K instructions. void TEST_CASE4(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; + const uintptr_t TN = 2, TM = 2, TK = KMAX(32, 1); - VSET(2, e16, m1); - VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A - VLOAD_16(v12, BF16_2_0, BF16_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A - VLOAD_16(v16, BF16_1_0, BF16_0_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B identity - VLOAD_16(v20, BF16_0_0, BF16_1_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B identity + VSET(2, e32, m1); + VLOAD_32(v8, 0x3F800000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 0.0] + VLOAD_32(v10, 0x00000000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [0.0, 1.0] + VLOAD_32(v12, 0x3F800000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=2 A: [1.0, 1.0] + VLOAD_32(v14, 0x40000000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=3 A: [2.0, 0.0] + VLOAD_32(v16, 0x3F800000, 0x40000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [1.0, 2.0] + VLOAD_32(v18, 0x40400000, 0x40800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [3.0, 4.0] + VLOAD_32(v20, 0x3F800000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=2 B: [1.0, 1.0] + VLOAD_32(v22, 0x3F000000, 0x3F000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=3 B: [0.5, 0.5] - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt4" ::: "memory"); - asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); + asm volatile("vtzero mt0" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v10, v18" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v12, v20" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v14, v22" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(7, v1, 0x3F800000, 0x40000000); // row0: [1.0, 2.0] - VCMP_U32(8, v2, 0x40400000, 0x40800000); // row1: [3.0, 4.0] + VTFMM_CMP_U32(27, v1, 0x40400000, 0x40800000); // row0: [3.0, 4.0] + VTFMM_CMP_U32(28, v2, 0x40800000, 0x40A00000); // row1: [4.0, 5.0] asm volatile("vtdiscard" ::: "memory"); } -// TC5: vtfmm.alt.tvv SEW=16, BF16→fp32, known dot product — C[0]=[17.0,23.0] C[1]=[39.0,53.0] +// TC5: FP32 accumulation interleaved between two destination tiles. void TEST_CASE5(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; + const uintptr_t TN = 2, TM = 2; - VSET(2, e16, m1); - VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A - VLOAD_16(v12, BF16_2_0, BF16_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A - VLOAD_16(v16, BF16_5_0, BF16_7_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B - VLOAD_16(v20, BF16_6_0, BF16_8_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B + VSET(2, e32, m1); + VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: [1.0, 3.0] + VLOAD_32(v12, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A dup (k=1 of (a)) + VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: [5.0, 7.0] + VLOAD_32(v20, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B dup (k=1 of (a)) + VLOAD_32(v24, 0x40000000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A': [2.0, 0.0] + VLOAD_32(v26, 0x3F800000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B': [1.0, 1.0] - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); + asm volatile("msettk x0, %0" :: "r"((uintptr_t)KMAX(32, 1)) : "memory"); + asm volatile("vtzero mt0" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v12, v20" ::: "memory"); // Complete the second K-step. + + asm volatile("msettk x0, %0" :: "r"((uintptr_t)KMAX(32, 1)) : "memory"); asm volatile("vtzero mt4" ::: "memory"); - asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); + asm volatile("vtfmm.tvv mt4, v24, v26" ::: "memory"); // (b) different tile + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); // (c) back to mt0 - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v3, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v4, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(9, v1, 0x41880000, 0x41B80000); // row0: [17.0, 23.0] - VCMP_U32(10, v2, 0x421C0000, 0x42540000); // row1: [39.0, 53.0] + VTFMM_CMP_U32(29, v1, 0x41700000, 0x41A80000); // mt0 row0: [15.0, 21.0] + VTFMM_CMP_U32(30, v2, 0x42340000, 0x427C0000); // mt0 row1: [45.0, 63.0] + VTFMM_CMP_U32(31, v3, 0x40000000, 0x40000000); // mt4 row0: [2.0, 2.0] + VTFMM_CMP_U32(32, v4, 0x00000000, 0x00000000); // mt4 row1: [0.0, 0.0] asm volatile("vtdiscard" ::: "memory"); } -// TC6: vtfmm.alt.tvv SEW=16, BF16→fp32, accumulation (×2) — C[0]=[2.0,4.0] C[1]=[6.0,8.0] +// TC6: FP32 identity product with two explicit single-K instructions. void TEST_CASE6(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; + const uintptr_t TN = 2, TM = 2, TK = KMAX(32, 1); - VSET(2, e16, m1); - VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_16(v12, BF16_2_0, BF16_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_16(v16, BF16_1_0, BF16_0_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_16(v20, BF16_0_0, BF16_1_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VSET(2, e32, m1); + VLOAD_32(v8, 0x3F800000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 0.0] + VLOAD_32(v12, 0x00000000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [0.0, 1.0] + VLOAD_32(v16, 0x3F800000, 0x40000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [1.0, 2.0] + VLOAD_32(v20, 0x40400000, 0x40800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [3.0, 4.0] - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt4" ::: "memory"); - asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); - asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); + asm volatile("vtzero mt0" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v12, v20" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(11, v1, 0x40000000, 0x40800000); // row0: [2.0, 4.0] - VCMP_U32(12, v2, 0x40C00000, 0x41000000); // row1: [6.0, 8.0] + VTFMM_CMP_U32(33, v1, 0x3F800000, 0x40000000); // row0: [1.0, 2.0] + VTFMM_CMP_U32(34, v2, 0x40400000, 0x40800000); // row1: [3.0, 4.0] asm volatile("vtdiscard" ::: "memory"); } -// TC7: vtfmm.tvv SEW=32, fp32×fp32→fp32 — C[0]=[5.0,7.0] C[1]=[15.0,21.0] +// TC7: FP32 full 8x8 tile accumulation with two explicit K steps. void TEST_CASE7(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; + const uintptr_t TN = 8, TM = 8, TK = KMAX(32, 1); - VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // [1.0, 3.0] - VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // [5.0, 7.0] + static float A[8][2] = { + {1, 10}, {2, 20}, {3, 30}, {4, 40}, + {5, 50}, {6, 60}, {7, 70}, {8, 80}, + }; + static float B[2][8] = { + {1, 1, 1, 1, 1, 1, 1, 1}, + {2, 2, 2, 2, 2, 2, 2, 2}, + }; + uintptr_t a_stride = 2 * sizeof(float); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("vsetvli zero, %0, e32, m1, ta, ma" :: "r"(TM) : "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtzero mt8" ::: "memory"); + + asm volatile("vlse32.v v16, (%0), %1" :: "r"(&A[0][0]), "r"(a_stride) : "v16", "memory"); + asm volatile("vlse32.v v20, (%0), %1" :: "r"(&A[0][1]), "r"(a_stride) : "v20", "memory"); + asm volatile("vle32.v v24, (%0)" :: "r"(&B[0][0]) : "v24", "memory"); + asm volatile("vle32.v v28, (%0)" :: "r"(&B[1][0]) : "v28", "memory"); + asm volatile("vtfmm.tvv mt8, v16, v24" ::: "memory"); + asm volatile("vtfmm.tvv mt8, v20, v28" ::: "memory"); + + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); + asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(8, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "v1", "memory"); + tss = TSS_ROW(8, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "v2", "memory"); + tss = TSS_ROW(8, 2); asm volatile("vtmv.v.t v3, %[t]" :: [t]"r"(tss) : "v3", "memory"); + tss = TSS_ROW(8, 3); asm volatile("vtmv.v.t v4, %[t]" :: [t]"r"(tss) : "v4", "memory"); + tss = TSS_ROW(8, 4); asm volatile("vtmv.v.t v5, %[t]" :: [t]"r"(tss) : "v5", "memory"); + tss = TSS_ROW(8, 5); asm volatile("vtmv.v.t v6, %[t]" :: [t]"r"(tss) : "v6", "memory"); + tss = TSS_ROW(8, 6); asm volatile("vtmv.v.t v7, %[t]" :: [t]"r"(tss) : "v7", "memory"); + tss = TSS_ROW(8, 7); asm volatile("vtmv.v.t v9, %[t]" :: [t]"r"(tss) : "v9", "memory"); - VCMP_U32(13, v1, 0x40A00000, 0x40E00000); // row0: [5.0, 7.0] - VCMP_U32(14, v2, 0x41700000, 0x41A80000); // row1: [15.0, 21.0] + // row m value = (m+1)*21, broadcast across all 8 columns. + VTFMM_CMP_U32(35, v1, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000); // row0: 21 + VTFMM_CMP_U32(36, v2, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000); // row1: 42 + VTFMM_CMP_U32(37, v3, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000); // row2: 63 + VTFMM_CMP_U32(38, v4, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000); // row3: 84 + VTFMM_CMP_U32(39, v5, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000); // row4: 105 + VTFMM_CMP_U32(40, v6, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000); // row5: 126 + VTFMM_CMP_U32(41, v7, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000); // row6: 147 + VTFMM_CMP_U32(42, v9, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000); // row7: 168 asm volatile("vtdiscard" ::: "memory"); } -// TC8: vtfmm.tvv SEW=32, accumulation (×2) — C[0]=[10.0,14.0] C[1]=[30.0,42.0] +// TC8: vtfmm.tvv SEW=16, fp16→fp32, A*I=A — C[0]=[1.0,2.0] C[1]=[3.0,4.0] void TEST_CASE8(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); - VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + // Spatz SDOTP reads {odd K, even K} from each base register word. + VSET(2, e32, m1); + VLOAD_32(v8, 0x40003C00, 0x44004200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: [1,2], [3,4] + VLOAD_32(v16, 0x00003C00, 0x3C000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: [1,0], [0,1] - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 2, 1, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); + asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(15, v1, 0x41200000, 0x41600000); // row0: [10.0, 14.0] - VCMP_U32(16, v2, 0x41F00000, 0x42280000); // row1: [30.0, 42.0] + VTFMM_CMP_U32(1, v1, 0x3F800000, 0x40000000); // row0: [1.0, 2.0] + VTFMM_CMP_U32(2, v2, 0x40400000, 0x40800000); // row1: [3.0, 4.0] asm volatile("vtdiscard" ::: "memory"); } -// TC9: vtfmm.tvv SEW=8, E4M3*E4M3 (altfmt=0) — C[0]=[5.0,7.0] C[1]=[15.0,21.0] +// TC9: vtfmm.tvv SEW=16, fp16→fp32, known dot product — C[0]=[17.0,23.0] C[1]=[39.0,53.0] void TEST_CASE9(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); - VSET(2, e8, m1); - VLOAD_8(v8, FP8_E4M3_1_0, FP8_E4M3_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A - VLOAD_8(v16, FP8_E4M3_5_0, FP8_E4M3_7_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B + VSET(2, e32, m1); + VLOAD_32(v8, 0x40003C00, 0x44004200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: [1,2], [3,4] + VLOAD_32(v16, 0x46004500, 0x48004700, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: [5,6], [7,8] - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); // clear alt_fmt (bit 8 in vtype) - asm volatile("msetmtypei 3, 0" ::: "memory"); + asm volatile("msetmtypei 2, 1, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -345,29 +368,29 @@ void TEST_CASE9(void) { tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(17, v1, 0x40A00000, 0x40E00000); // row0: [5.0, 7.0] - VCMP_U32(18, v2, 0x41700000, 0x41A80000); // row1: [15.0, 21.0] + VTFMM_CMP_U32(3, v1, 0x41880000, 0x41B80000); // row0: [17.0, 23.0] + VTFMM_CMP_U32(4, v2, 0x421C0000, 0x42540000); // row1: [39.0, 53.0] asm volatile("vtdiscard" ::: "memory"); } -// TC10: vtfmm.tvv SEW=8, E4M3*E5M2 (altfmt=1) — C[0]=[2.0,4.0] C[1]=[4.0,8.0] +// TC10: vtfmm.tvv SEW=16, fp16→fp32, accumulation (×2) — C[0]=[2.0,4.0] C[1]=[6.0,8.0] void TEST_CASE10(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); - VSET(2, e8, m1); - VLOAD_8(v8, FP8_E4M3_1_0, FP8_E4M3_2_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: E4M3 - VLOAD_8(v16, FP8_E5M2_2_0, FP8_E5M2_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: E5M2 + VSET(2, e32, m1); + VLOAD_32(v8, 0x40003C00, 0x44004200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VLOAD_32(v16, 0x00003C00, 0x3C000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); // set alt_fmt (bit 8 in vtype) - asm volatile("msetmtypei 3, 0" ::: "memory"); + asm volatile("msetmtypei 2, 1, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -375,30 +398,62 @@ void TEST_CASE10(void) { tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(19, v1, 0x40000000, 0x40800000); // row0: [2.0, 4.0] - VCMP_U32(20, v2, 0x40800000, 0x41000000); // row1: [4.0, 8.0] + VTFMM_CMP_U32(5, v1, 0x40000000, 0x40800000); // row0: [2.0, 4.0] + VTFMM_CMP_U32(6, v2, 0x40C00000, 0x41000000); // row1: [6.0, 8.0] - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); asm volatile("vtdiscard" ::: "memory"); } -// TC11: vtfmm.alt.tvv SEW=8, E5M2*E4M3 (altfmt=0) — C[0]=[3.0,4.0] C[1]=[6.0,8.0] +// TC11: Packed FP16 K lanes across the compute-edge row and column boundary. void TEST_CASE11(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; + const uintptr_t TN = 9, TM = 9, TK = KMAX(16, 2); - VSET(2, e8, m1); - VLOAD_8(v8, FP8_E5M2_1_0, FP8_E5M2_2_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: E5M2 - VLOAD_8(v16, FP8_E4M3_3_0, FP8_E4M3_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: E4M3 + VSET(9, e32, m1); + VLOAD_32(v8, 0x40003C00,0x40004000,0x40004200,0x40004400,0x40004500, + 0x40004600,0x40004700,0x40004800,0x40004880,0,0,0,0,0,0,0); + VLOAD_32(v16, 0x48803C00,0x48004000,0x47004200,0x46004400,0x45004500, + 0x44004600,0x42004700,0x40004800,0x3C004880,0,0,0,0,0,0,0); - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("msetmtypei 3, 0" ::: "memory"); + asm volatile("msetmtypei 2, 1, 0" ::: "memory"); + asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); + asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); + asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); + asm volatile("vtzero mt0" ::: "memory"); + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); + asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); + + uintptr_t tss; + tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(0, 8); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + VTFMM_CMP_U32(43, v1, 0x41980000, 0x41900000, 0x41880000, 0x41800000, + 0x41700000, 0x41600000, 0x41500000, 0x41400000, 0x41300000); + VTFMM_CMP_U32(44, v2, 0x41D80000, 0x42080000, 0x42240000, 0x42400000, + 0x425C0000, 0x42780000, 0x428A0000, 0x42980000, 0x42A60000); + + asm volatile("vtdiscard" ::: "memory"); +} + +// TC12: vtfmm.alt.tvv SEW=16, BF16→fp32, A*I=A — C[0]=[1.0,2.0] C[1]=[3.0,4.0] +void TEST_CASE12(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); + + VSET(2, e16, m1); + VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A + VLOAD_16(v12, BF16_2_0, BF16_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A + VLOAD_16(v16, BF16_1_0, BF16_0_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B identity + VLOAD_16(v20, BF16_0_0, BF16_1_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B identity + + asm volatile("msetmtypei 2, 1, 1" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt4" ::: "memory"); asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -406,15 +461,15 @@ void TEST_CASE11(void) { tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(21, v1, 0x40400000, 0x40800000); // row0: [3.0, 4.0] - VCMP_U32(22, v2, 0x40C00000, 0x41000000); // row1: [6.0, 8.0] + VTFMM_CMP_U32(7, v1, 0x3F800000, 0x40000000); // row0: [1.0, 2.0] + VTFMM_CMP_U32(8, v2, 0x40400000, 0x40800000); // row1: [3.0, 4.0] asm volatile("vtdiscard" ::: "memory"); } -// TC12: vtfmm.alt.tvv SEW=16, BF16 altfmt isolation — altfmt=1 ignored for SEW=16 -void TEST_CASE12(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; +// TC13: vtfmm.alt.tvv SEW=16, BF16→fp32, known dot product — C[0]=[17.0,23.0] C[1]=[39.0,53.0] +void TEST_CASE13(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); VSET(2, e16, m1); VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A @@ -422,15 +477,14 @@ void TEST_CASE12(void) { VLOAD_16(v16, BF16_5_0, BF16_7_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B VLOAD_16(v20, BF16_6_0, BF16_8_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("msetmtypei 2, 1" ::: "memory"); + asm volatile("msetmtypei 2, 1, 1" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt4" ::: "memory"); asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -438,158 +492,128 @@ void TEST_CASE12(void) { tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(23, v1, 0x41880000, 0x41B80000); // row0: [17.0, 23.0] - VCMP_U32(24, v2, 0x421C0000, 0x42540000); // row1: [39.0, 53.0] + VTFMM_CMP_U32(9, v1, 0x41880000, 0x41B80000); // row0: [17.0, 23.0] + VTFMM_CMP_U32(10, v2, 0x421C0000, 0x42540000); // row1: [39.0, 53.0] - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); asm volatile("vtdiscard" ::: "memory"); } -// TC13: vtfmm.tvv SEW=32, vke=1, tk=2 fused. Same A/B values duplicated at -// both K-rows (v8/v12 stride=8/2=4, v16/v20 same stride), so the fused -// two-beat result must exactly match TC8's two-separate-dispatch -// accumulation of the identical inputs — C[0]=[10.0,14.0] C[1]=[30.0,42.0]. -void TEST_CASE13(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; +// TC14: vtfmm.alt.tvv SEW=16, BF16→fp32, accumulation (×2) — C[0]=[2.0,4.0] C[1]=[6.0,8.0] +void TEST_CASE14(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); - VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 3.0] - VLOAD_32(v12, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [1.0, 3.0] - VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [5.0, 7.0] - VLOAD_32(v20, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [5.0, 7.0] + VSET(2, e16, m1); + VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VLOAD_16(v12, BF16_2_0, BF16_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VLOAD_16(v16, BF16_1_0, BF16_0_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); + VLOAD_16(v20, BF16_0_0, BF16_1_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - asm volatile("msetmtypei 5, 2" ::: "memory"); // uimm5=5: vke=1, mtwiden=1 + asm volatile("msetmtypei 2, 1, 1" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtzero mt4" ::: "memory"); + asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); + asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(25, v1, 0x41200000, 0x41600000); // row0: [10.0, 14.0] - VCMP_U32(26, v2, 0x41F00000, 0x42280000); // row1: [30.0, 42.0] + VTFMM_CMP_U32(11, v1, 0x40000000, 0x40800000); // row0: [2.0, 4.0] + VTFMM_CMP_U32(12, v2, 0x40C00000, 0x41000000); // row1: [6.0, 8.0] asm volatile("vtdiscard" ::: "memory"); } -// TC14: vtfmm.tvv SEW=32, vke=1, tk=4 fused, four distinct K-rows (stride -// 8/4=2: v8/v10/v12/v14, v16/v18/v20/v22). Hand-computed: -// A = [[1,0],[0,1],[1,1],[2,0]] B = [[1,2],[3,4],[1,1],[0.5,0.5]] -// C[m,n] = sum_k A[k,m]*B[k,n] -> row0=[3,4] row1=[4,5] -void TEST_CASE14(void) { - const uintptr_t TN = 2, TM = 2, TK = 4; +// TC15: vtfmm.alt.tvv SEW=16, BF16 with altfmt=1. +void TEST_CASE15(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(16, 2); - VLOAD_32(v8, 0x3F800000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 0.0] - VLOAD_32(v10, 0x00000000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [0.0, 1.0] - VLOAD_32(v12, 0x3F800000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=2 A: [1.0, 1.0] - VLOAD_32(v14, 0x40000000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=3 A: [2.0, 0.0] - VLOAD_32(v16, 0x3F800000, 0x40000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [1.0, 2.0] - VLOAD_32(v18, 0x40400000, 0x40800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [3.0, 4.0] - VLOAD_32(v20, 0x3F800000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=2 B: [1.0, 1.0] - VLOAD_32(v22, 0x3F000000, 0x3F000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=3 B: [0.5, 0.5] + VSET(2, e16, m1); + VLOAD_16(v8, BF16_1_0, BF16_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A + VLOAD_16(v12, BF16_2_0, BF16_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A + VLOAD_16(v16, BF16_5_0, BF16_7_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B + VLOAD_16(v20, BF16_6_0, BF16_8_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B - asm volatile("msetmtypei 5, 2" ::: "memory"); // uimm5=5: vke=1, mtwiden=1 + asm volatile("msetmtypei 2, 1, 1" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); + asm volatile("vtzero mt4" ::: "memory"); + asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(27, v1, 0x40400000, 0x40800000); // row0: [3.0, 4.0] - VCMP_U32(28, v2, 0x40800000, 0x40A00000); // row1: [4.0, 5.0] + VTFMM_CMP_U32(23, v1, 0x41880000, 0x41B80000); // row0: [17.0, 23.0] + VTFMM_CMP_U32(24, v2, 0x421C0000, 0x42540000); // row1: [39.0, 53.0] asm volatile("vtdiscard" ::: "memory"); } -// TC15: sequencing/hazard check around a fused instruction. Dispatches, in -// order: (a) tk=2 fused vtfmm to mt0 (A=[1,3],B=[5,7], same as TC13); (b) a -// plain tk=1 vtfmm to a DIFFERENT tile mt4 (A'=[2,0],B'=[1,1]); (c) another -// plain tk=1 vtfmm back to mt0 (same A,B as (a)). Note: path_ready[GRP_MAC] -// is a group-wide (not per-tile) admission gate, so (b) cannot actually -// dispatch until (a)'s beat sequence fully retires -- there is no runtime -// interleaving to observe here. What this test verifies instead is that -// the sequence produces exactly correct results for BOTH tiles: mt0 must -// accumulate all three contributions (2 from (a)'s fused beats + 1 from -// (c)) with none lost or double-counted, and mt4 must be unaffected by -// mt0's activity. mt0 = 3x(A^T.B) = row0:[15,21] row1:[45,63]. -// mt4 = A'^T.B' = row0:[2,2] row1:[0,0]. -void TEST_CASE15(void) { - const uintptr_t TN = 2, TM = 2; +// TC16: vtfmm.tvv SEW=8, E4M3*E4M3 (altfmt=0) — C[0]=[5.0,7.0] C[1]=[15.0,21.0] +void TEST_CASE16(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(8, 4); - VLOAD_32(v8, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: [1.0, 3.0] - VLOAD_32(v12, 0x3F800000, 0x40400000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A dup (k=1 of (a)) - VLOAD_32(v16, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: [5.0, 7.0] - VLOAD_32(v20, 0x40A00000, 0x40E00000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B dup (k=1 of (a)) - VLOAD_32(v24, 0x40000000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A': [2.0, 0.0] - VLOAD_32(v26, 0x3F800000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B': [1.0, 1.0] + VSET(2, e8, m1); + // Zero unused K inputs before loading the test operands. + asm volatile("vmv.v.i v10, 0; vmv.v.i v12, 0; vmv.v.i v14, 0; " + "vmv.v.i v18, 0; vmv.v.i v20, 0; vmv.v.i v22, 0" + ::: "v10", "v12", "v14", "v18", "v20", "v22", "memory"); + VLOAD_8(v8, FP8_E4M3_1_0, FP8_E4M3_3_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A + VLOAD_8(v16, FP8_E4M3_5_0, FP8_E4M3_7_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B - asm volatile("msetmtypei 5, 2" ::: "memory"); // uimm5=5: vke=1, mtwiden=1 + asm volatile("msetmtypei 3, 0, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"((uintptr_t)2) : "memory"); + asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); // (a) tk=2 fused - - asm volatile("msettk x0, %0" :: "r"((uintptr_t)1) : "memory"); - asm volatile("vtzero mt4" ::: "memory"); - asm volatile("vtfmm.tvv mt4, v24, v26" ::: "memory"); // (b) different tile - asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); // (c) back to mt0 + asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v3, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v4, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(29, v1, 0x41700000, 0x41A80000); // mt0 row0: [15.0, 21.0] - VCMP_U32(30, v2, 0x42340000, 0x427C0000); // mt0 row1: [45.0, 63.0] - VCMP_U32(31, v3, 0x40000000, 0x40000000); // mt4 row0: [2.0, 2.0] - VCMP_U32(32, v4, 0x00000000, 0x00000000); // mt4 row1: [0.0, 0.0] + VTFMM_CMP_U32(17, v1, 0x40A00000, 0x40E00000); // row0: [5.0, 7.0] + VTFMM_CMP_U32(18, v2, 0x41700000, 0x41A80000); // row1: [15.0, 21.0] asm volatile("vtdiscard" ::: "memory"); } -// TC16: vtfmm.tvv SEW=32, vke=1, tk=2 fused, two DISTINCT K-rows (stride -// 8/2=4: v8/v12, v16/v20). TC13 only ever used identical data duplicated -// at both k-rows, which cannot detect a beat-offset/ordering bug (any -// mixup between the two beats is invisible when both contribute the same -// value). Hand-computed: -// A = [[1,0],[0,1]] B = [[1,2],[3,4]] -// C[m,n] = sum_k A[k,m]*B[k,n] -> row0=[1,2] row1=[3,4] -void TEST_CASE16(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; +// TC17: vtfmm.tvv SEW=8, E4M3*E5M2 (altfmt=1) — C[0]=[2.0,4.0] C[1]=[4.0,8.0] +void TEST_CASE17(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(8, 4); - VLOAD_32(v8, 0x3F800000, 0x00000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A: [1.0, 0.0] - VLOAD_32(v12, 0x00000000, 0x3F800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A: [0.0, 1.0] - VLOAD_32(v16, 0x3F800000, 0x40000000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B: [1.0, 2.0] - VLOAD_32(v20, 0x40400000, 0x40800000, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B: [3.0, 4.0] + VSET(2, e8, m1); + // Zero unused K inputs before loading the test operands. + asm volatile("vmv.v.i v10, 0; vmv.v.i v12, 0; vmv.v.i v14, 0; " + "vmv.v.i v18, 0; vmv.v.i v20, 0; vmv.v.i v22, 0" + ::: "v10", "v12", "v14", "v18", "v20", "v22", "memory"); + VLOAD_8(v8, FP8_E4M3_1_0, FP8_E4M3_2_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: E4M3 + VLOAD_8(v16, FP8_E5M2_2_0, FP8_E5M2_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: E5M2 - asm volatile("msetmtypei 5, 2" ::: "memory"); // uimm5=5: vke=1, mtwiden=1 + asm volatile("msetmtypei 3, 0, 1" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -597,104 +621,71 @@ void TEST_CASE16(void) { tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - VCMP_U32(33, v1, 0x3F800000, 0x40000000); // row0: [1.0, 2.0] - VCMP_U32(34, v2, 0x40400000, 0x40800000); // row1: [3.0, 4.0] + VTFMM_CMP_U32(19, v1, 0x40000000, 0x40800000); // row0: [2.0, 4.0] + VTFMM_CMP_U32(20, v2, 0x40800000, 0x41000000); // row1: [4.0, 8.0] asm volatile("vtdiscard" ::: "memory"); } -// TC17: vtfmm.tvv SEW=32, vke=1, tk=2 fused, TM=TN=8 (matches TE=8, same -// size as the real gemm.c kernel -- TC16 only used TM=2 and passed, but a -// real gemm.c M64_N64_K64 run with a TK_FUSE=2 kernel produced wrong -// results specifically on odd tile-rows of mt8, fed by strided -// (vlse32.v) loads into v16/v20 -- mirrors gemm.c's exact register -// choice for its "a1" operand and its exact destination tile, to isolate -// whether this is a TM=8-specific RTL bug or a gemm.c bug. -// A[m][k], m=0..7, k=0..1: A[m][0]=m+1, A[m][1]=(m+1)*10. -// B[k][n]=1 for k=0, =2 for k=1 (same for all n=0..7). -// C[m][n] = A[m][0]*1 + A[m][1]*2 = (m+1) + (m+1)*20 = (m+1)*21, all n. -// -// NOT CALLED from main(): this hangs the simulator outright (times out, -// no PASS/FAIL ever printed) rather than producing a wrong-but-terminating -// result -- a second, unresolved bug distinct from the value corruption -// this test was written to isolate. Left in place, uncalled, as a starting -// point for whoever picks up the TK_FUSE=2 investigation next. -void TEST_CASE17(void) { - const uintptr_t TN = 8, TM = 8, TK = 2; +// TC18: vtfmm.alt.tvv SEW=8, E5M2*E4M3 (altfmt=0) — C[0]=[3.0,4.0] C[1]=[6.0,8.0] +void TEST_CASE18(void) { + const uintptr_t TN = 2, TM = 2, TK = KMAX(8, 4); - static float A[8][2] = { - {1, 10}, {2, 20}, {3, 30}, {4, 40}, - {5, 50}, {6, 60}, {7, 70}, {8, 80}, - }; - static float B[2][8] = { - {1, 1, 1, 1, 1, 1, 1, 1}, - {2, 2, 2, 2, 2, 2, 2, 2}, - }; - uintptr_t a_stride = 2 * sizeof(float); + VSET(2, e8, m1); + // Zero unused K inputs before loading the test operands. + asm volatile("vmv.v.i v10, 0; vmv.v.i v12, 0; vmv.v.i v14, 0; " + "vmv.v.i v18, 0; vmv.v.i v20, 0; vmv.v.i v22, 0" + ::: "v10", "v12", "v14", "v18", "v20", "v22", "memory"); + VLOAD_8(v8, FP8_E5M2_1_0, FP8_E5M2_2_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // A: E5M2 + VLOAD_8(v16, FP8_E4M3_3_0, FP8_E4M3_4_0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // B: E4M3 - asm volatile("vsetvli zero, %0, e32, m1, ta, ma" :: "r"(TM) : "memory"); - asm volatile("msetmtypei 5, 2" ::: "memory"); + asm volatile("msetmtypei 3, 0, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt8" ::: "memory"); - - asm volatile("vlse32.v v16, (%0), %1" :: "r"(&A[0][0]), "r"(a_stride) : "v16", "memory"); - asm volatile("vlse32.v v20, (%0), %1" :: "r"(&A[0][1]), "r"(a_stride) : "v20", "memory"); - asm volatile("vle32.v v24, (%0)" :: "r"(&B[0][0]) : "v24", "memory"); - asm volatile("vle32.v v28, (%0)" :: "r"(&B[1][0]) : "v28", "memory"); - asm volatile("vtfmm.tvv mt8, v16, v24" ::: "memory"); + asm volatile("vtzero mt4" ::: "memory"); + asm volatile("vtfmm.alt.tvv mt4, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); uintptr_t tss; - tss = TSS_ROW(8, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "v1", "memory"); - tss = TSS_ROW(8, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "v2", "memory"); - tss = TSS_ROW(8, 2); asm volatile("vtmv.v.t v3, %[t]" :: [t]"r"(tss) : "v3", "memory"); - tss = TSS_ROW(8, 3); asm volatile("vtmv.v.t v4, %[t]" :: [t]"r"(tss) : "v4", "memory"); - tss = TSS_ROW(8, 4); asm volatile("vtmv.v.t v5, %[t]" :: [t]"r"(tss) : "v5", "memory"); - tss = TSS_ROW(8, 5); asm volatile("vtmv.v.t v6, %[t]" :: [t]"r"(tss) : "v6", "memory"); - tss = TSS_ROW(8, 6); asm volatile("vtmv.v.t v7, %[t]" :: [t]"r"(tss) : "v7", "memory"); - tss = TSS_ROW(8, 7); asm volatile("vtmv.v.t v9, %[t]" :: [t]"r"(tss) : "v9", "memory"); + tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); + tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - // row m value = (m+1)*21, broadcast across all 8 columns. - VCMP_U32(35, v1, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000, 0x41A80000); // row0: 21 - VCMP_U32(36, v2, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000, 0x42280000); // row1: 42 - VCMP_U32(37, v3, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000, 0x427C0000); // row2: 63 - VCMP_U32(38, v4, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000, 0x42A80000); // row3: 84 - VCMP_U32(39, v5, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000, 0x42D20000); // row4: 105 - VCMP_U32(40, v6, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000, 0x42FC0000); // row5: 126 - VCMP_U32(41, v7, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000, 0x43130000); // row6: 147 - VCMP_U32(42, v9, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000, 0x43280000); // row7: 168 + VTFMM_CMP_U32(21, v1, 0x40400000, 0x40800000); // row0: [3.0, 4.0] + VTFMM_CMP_U32(22, v2, 0x40C00000, 0x41000000); // row1: [6.0, 8.0] asm volatile("vtdiscard" ::: "memory"); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_fp(); - enable_vme(); - - // TEST_CASE1(); // vtfmm.tvv SEW=16 fp16→fp32, A*I - // TEST_CASE2(); // vtfmm.tvv SEW=16 fp16→fp32, known product - // TEST_CASE3(); // vtfmm.tvv SEW=16 fp16→fp32, accumulation ×2 - // TEST_CASE4(); // vtfmm.alt SEW=16 BF16→fp32, A*I - // TEST_CASE5(); // vtfmm.alt SEW=16 BF16→fp32, known product - // TEST_CASE6(); // vtfmm.alt SEW=16 BF16→fp32, accumulation ×2 - TEST_CASE7(); // vtfmm.tvv SEW=32 fp32→fp32, known product - TEST_CASE8(); // vtfmm.tvv SEW=32 fp32→fp32, accumulation ×2 - // TEST_CASE9(); // vtfmm.tvv SEW=8 E4M3*E4M3 altfmt=0 - // TEST_CASE10(); // vtfmm.tvv SEW=8 E4M3*E5M2 altfmt=1 - // TEST_CASE11(); // vtfmm.alt SEW=8 E5M2*E4M3 altfmt=0 - // TEST_CASE12(); // vtfmm.alt SEW=16 BF16 altfmt isolation - TEST_CASE13(); // vtfmm.tvv SEW=32 vke=1 tk=2 fused - TEST_CASE14(); // vtfmm.tvv SEW=32 vke=1 tk=4 fused - TEST_CASE15(); // vtfmm.tvv SEW=32 vke=1, sequencing/hazard check - TEST_CASE16(); // vtfmm.tvv SEW=32 vke=1 tk=2 fused, distinct K-rows - // TEST_CASE17(); // vtfmm.tvv SEW=32 vke=1 tk=2 fused, TM=TN=8, mt8/v16-v20 -- hangs, see comment above the function + + // FP32 -> FP32. + TEST_CASE1(); // Known product. + TEST_CASE2(); // Accumulation by two. + TEST_CASE3(); // Two explicit K steps. + TEST_CASE4(); // Four explicit K steps. + TEST_CASE5(); // Interleaved tile accumulation. + TEST_CASE6(); // Identity from two K steps. + // TEST_CASE7(); // Full 8x8 tile; enable after hardware validation. + + // FP16 -> FP32. + TEST_CASE8(); // Identity product. + TEST_CASE9(); // Known product. + TEST_CASE10(); // Accumulation by two. + TEST_CASE11(); // Packed K lanes across the compute-edge boundary. + + // Additional Zvt v0.4 formats; enable with matching hardware support. + // TEST_CASE12(); // BF16 identity product. + // TEST_CASE13(); // BF16 known product. + // TEST_CASE14(); // BF16 accumulation by two. + // TEST_CASE15(); // BF16 altfmt selection. + // TEST_CASE16(); // E4M3 * E4M3. + // TEST_CASE17(); // E4M3 * E5M2. + // TEST_CASE18(); // E5M2 * E4M3. EXIT_CHECK(); } diff --git a/sw/vme/isaTests/vtle.c b/sw/vme/isaTests/vtle.c index 7affa2cae..a22c3905c 100644 --- a/sw/vme/isaTests/vtle.c +++ b/sw/vme/isaTests/vtle.c @@ -2,250 +2,156 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// vtle.c — ISA tests for vtle8, vtle16, vtle32, vtle64 -// -// ------------------------------------------------------- // Test cases: -// TC1: vtle8 row load → uint8 zero-extended to int32 -// TC2: vtle16 row load → uint16 zero-extended to int32 -// TC3: vtle32 row load → int32 direct -// TC4: vtle8 col load (pattern=1) -// TC5: vtle8 all rows of 4×4 tile -// TC6: vtle8 tile isolation (mt0 loaded, mt4 must stay zero) -// TC7: vtle32 round-trip with vtse32 +// TC1-TC2: 8-bit row and column loads. +// TC3-TC4: 16-bit row and column loads. +// TC5-TC6: 32-bit row and column loads. #include #include "vector_macros.h" #include "vector_matrix_macros.h" -// TC1: vtle8 row load — uint8 zero-extended to int32 -void TEST_CASE1(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - uint8_t tmp[] = {10, 20, 30, 40}; - uint8_t *src = (uint8_t *)snrt_l1alloc(4); memcpy(src, tmp, 4); - - uintptr_t tss = TSS_ROW(0, 0); - uintptr_t base = (uintptr_t)src; - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - uintptr_t t = TSS_ROW(0, 0); - asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(t) : "memory"); - VCMP_I32(1, v1, 10, 20, 30, 40); - - asm volatile("vtdiscard" ::: "memory"); -} - -// TC2: vtle16 row load — uint16 zero-extended to int32 -void TEST_CASE2(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - uint16_t tmp[] = {100, 200, 300, 400}; - uint16_t *src = (uint16_t *)snrt_l1alloc(8); memcpy(src, tmp, 8); - - uintptr_t tss = TSS_ROW(0, 0); - uintptr_t base = (uintptr_t)src; - - asm volatile("msetmtypei 2, 1" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); +static void TEST_CASE1(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_ROW(0, 0); + uint8_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint8_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint8_t)(~(uint8_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 0, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(10, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtle16 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - uintptr_t t = TSS_ROW(0, 0); - asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(t) : "memory"); - VCMP_I32(2, v1, 100, 200, 300, 400); - + asm volatile("vtle8 %0, (%1)" :: "r"(tss), "r"(src) : "memory"); + asm volatile("vtmv.v.t v8, %0" :: "r"(tss) : "v8", "memory"); + asm volatile("vse8.v v8, (%0)" :: "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(11, dst[i + 1], src[i]); } + XCMP(12, dst[0], 0x55); + XCMP(13, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC3: vtle32 row load — int32 direct (including negatives) -void TEST_CASE3(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - int32_t tmp[] = {1000, 2000, -3000, -4000}; - int32_t *src = (int32_t *)snrt_l1alloc(16); memcpy(src, tmp, 16); - - uintptr_t tss = TSS_ROW(0, 0); - uintptr_t base = (uintptr_t)src; - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); +static void TEST_CASE2(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_COL(0, 0); + uint8_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint8_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint8_t)(~(uint8_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 0, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(20, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtle32 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - uintptr_t t = TSS_ROW(0, 0); - asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(t) : "memory"); - VCMP_I32(3, v1, 1000, 2000, -3000, -4000); - + asm volatile("vtle8 %0, (%1)" :: "r"(tss), "r"(src) : "memory"); + asm volatile("vtmv.v.t v8, %0" :: "r"(tss) : "v8", "memory"); + asm volatile("vse8.v v8, (%0)" :: "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(21, dst[i + 1], src[i]); } + XCMP(22, dst[0], 0x55); + XCMP(23, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC4: vtle8 col load — pattern=1 (column) -void TEST_CASE4(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - uint8_t tmp[] = {5, 6, 7, 8}; - uint8_t *src = (uint8_t *)snrt_l1alloc(4); memcpy(src, tmp, 4); - - uintptr_t tss = TSS_COL(0, 0); - uintptr_t base = (uintptr_t)src; - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); +static void TEST_CASE3(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_ROW(0, 0); + uint16_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint16_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint16_t)(~(uint16_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 1, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(30, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - uintptr_t t = TSS_COL(0, 0); - asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(t) : "memory"); - VCMP_I32(4, v1, 5, 6, 7, 8); - + asm volatile("vtle16 %0, (%1)" :: "r"(tss), "r"(src) : "memory"); + asm volatile("vtmv.v.t v8, %0" :: "r"(tss) : "v8", "memory"); + asm volatile("vse16.v v8, (%0)" :: "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(31, dst[i + 1], src[i]); } + XCMP(32, dst[0], 0x55); + XCMP(33, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC5: vtle8 all rows — load all 4 rows from different buffers -void TEST_CASE5(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - uint8_t tmp0[4] = { 1, 2, 3, 4}; - uint8_t tmp1[4] = { 5, 6, 7, 8}; - uint8_t tmp2[4] = { 9, 10, 11, 12}; - uint8_t tmp3[4] = {13, 14, 15, 16}; - uint8_t *r0 = (uint8_t *)snrt_l1alloc(4); memcpy(r0, tmp0, 4); - uint8_t *r1 = (uint8_t *)snrt_l1alloc(4); memcpy(r1, tmp1, 4); - uint8_t *r2 = (uint8_t *)snrt_l1alloc(4); memcpy(r2, tmp2, 4); - uint8_t *r3 = (uint8_t *)snrt_l1alloc(4); memcpy(r3, tmp3, 4); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); +static void TEST_CASE4(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_COL(0, 0); + uint16_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint16_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint16_t)(~(uint16_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 1, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(40, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - - uintptr_t tss, base; - tss = TSS_ROW(0, 0); base = (uintptr_t)r0; - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - tss = TSS_ROW(0, 1); base = (uintptr_t)r1; - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - tss = TSS_ROW(0, 2); base = (uintptr_t)r2; - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - tss = TSS_ROW(0, 3); base = (uintptr_t)r3; - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t t; - t = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(t) : "memory"); - t = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(t) : "memory"); - t = TSS_ROW(0, 2); asm volatile("vtmv.v.t v3, %[t]" :: [t]"r"(t) : "memory"); - t = TSS_ROW(0, 3); asm volatile("vtmv.v.t v4, %[t]" :: [t]"r"(t) : "memory"); - - VCMP_I32(5, v1, 1, 2, 3, 4); - VCMP_I32(6, v2, 5, 6, 7, 8); - VCMP_I32(7, v3, 9, 10, 11, 12); - VCMP_I32(8, v4, 13, 14, 15, 16); - + asm volatile("vtle16 %0, (%1)" :: "r"(tss), "r"(src) : "memory"); + asm volatile("vtmv.v.t v8, %0" :: "r"(tss) : "v8", "memory"); + asm volatile("vse16.v v8, (%0)" :: "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(41, dst[i + 1], src[i]); } + XCMP(42, dst[0], 0x55); + XCMP(43, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC6: vtle8 tile isolation — mt0 loaded, mt4 must stay zero -void TEST_CASE6(void) { - const uintptr_t TN = 6, TM = 6, TK = 2; - - uint8_t tmp[] = {77, 88}; - uint8_t *src = (uint8_t *)snrt_l1alloc(2); memcpy(src, tmp, 2); - - uintptr_t tss = TSS_ROW(0, 0); - uintptr_t base = (uintptr_t)src; - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); +static void TEST_CASE5(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_ROW(0, 0); + uint32_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint32_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint32_t)(~(uint32_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(50, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtzero mt4" ::: "memory"); - asm volatile("vtle8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t t; - t = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(t) : "memory"); - t = TSS_ROW(4, 0); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(t) : "memory"); - - VCMP_I32(9, v1, 77, 88); // mt0: loaded - VCMP_I32(10, v2, 0, 0); // mt4: untouched - + asm volatile("vtle32 %0, (%1)" :: "r"(tss), "r"(src) : "memory"); + asm volatile("vtmv.v.t v8, %0" :: "r"(tss) : "v8", "memory"); + asm volatile("vse32.v v8, (%0)" :: "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(51, dst[i + 1], src[i]); } + XCMP(52, dst[0], 0x55); + XCMP(53, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC7: vtle32 + vtse32 round-trip -void TEST_CASE7(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - int32_t tmp[] = {111, 222, 333, 444}; - int32_t *src = (int32_t *)snrt_l1alloc(16); memcpy(src, tmp, 16); - int32_t *dst = (int32_t *)snrt_l1alloc(4 * sizeof(int32_t)); - dst[0] = 0; dst[1] = 0; dst[2] = 0; dst[3] = 0; - - uintptr_t tss = TSS_ROW(0, 0); - uintptr_t base; - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); +static void TEST_CASE6(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_COL(0, 0); + uint32_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint32_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint32_t)(~(uint32_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(60, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - - base = (uintptr_t)src; - asm volatile("vtle32 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - base = (uintptr_t)dst; - asm volatile("vtse32 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - asm volatile("vle32.v v1, (%0)" :: "r"(dst) : "memory"); - VCMP_I32(11, v1, 111, 222, 333, 444); - + asm volatile("vtle32 %0, (%1)" :: "r"(tss), "r"(src) : "memory"); + asm volatile("vtmv.v.t v8, %0" :: "r"(tss) : "v8", "memory"); + asm volatile("vse32.v v8, (%0)" :: "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(61, dst[i + 1], src[i]); } + XCMP(62, dst[0], 0x55); + XCMP(63, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); - - // TEST_CASE1(); - // TEST_CASE2(); + TEST_CASE1(); + TEST_CASE2(); TEST_CASE3(); - // TEST_CASE4(); - // TEST_CASE5(); - // TEST_CASE6(); - // TEST_CASE7(); - + TEST_CASE4(); + TEST_CASE5(); + TEST_CASE6(); EXIT_CHECK(); } diff --git a/sw/vme/isaTests/vtmm.c b/sw/vme/isaTests/vtmm.c index 1ecef8f5d..c38fdd6f9 100644 --- a/sw/vme/isaTests/vtmm.c +++ b/sw/vme/isaTests/vtmm.c @@ -2,456 +2,132 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// vtmm.c — ISA tests for vtmmu.tvv and vtmms.tvv -// -// ------------------------------------------------------- // Test cases: -// TC1: vtmmu A*I=A (uint8*uint8, identity) -// TC2: vtmmu known dot product -// TC3: vtmmu accumulation — call twice = 2x -// TC4: vtmmu tile isolation — mt0 and mt4 independent -// TC5: vtmmu altfmt=1 (uint8 × int8), A*I=A -// TC6: vtmmu altfmt=1 (uint8 × int8), B with negatives -// TC7: vtmmu altfmt isolation — B=0xFE as uint8 vs int8 -// TC8: vtmms int8*uint8 (ALTFMT=0) -// TC9: vtmms int8*int8 (ALTFMT=1) -// TC10: vtmms extreme values — sign extension check -// TC11: vtmms ALTFMT isolation +// TC1: vtmmu with unsigned A and unsigned B. +// TC2: vtmmu with unsigned A and signed B. +// TC3: vtmms with signed A and unsigned B. +// TC4: vtmms with signed A and signed B, read back by columns. #include + #include "vector_macros.h" #include "vector_matrix_macros.h" -// TC1: vtmmu A*I=A (uint8*uint8, identity) -void TEST_CASE1(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - - VSET(2, e8, m1); - VLOAD_8(v8, 1, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v10, 2, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 1, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v18, 0, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtmmu.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(1, v1, 1, 2); - VCMP_I32(2, v2, 3, 4); - - asm volatile("vtdiscard" ::: "memory"); -} - -// TC2: vtmmu known dot product — C[0]=[17,23], C[1]=[39,53] -void TEST_CASE2(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - - VSET(2, e8, m1); - VLOAD_8(v8, 1, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v10, 2, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 5, 7, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v18, 6, 8, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt12" ::: "memory"); - asm volatile("vtmmu.tvv mt12, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(12, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(12, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(3, v1, 17, 23); - VCMP_I32(4, v2, 39, 53); - - asm volatile("vtdiscard" ::: "memory"); -} - -// TC3: vtmmu accumulation — call twice = 2x, C[0]=[2,4], C[1]=[6,8] -void TEST_CASE3(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - - VSET(2, e8, m1); - VLOAD_8(v8, 1, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v10, 2, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 1, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v18, 0, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt12" ::: "memory"); - asm volatile("vtmmu.tvv mt12, v8, v16" ::: "memory"); - asm volatile("vtmmu.tvv mt12, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(12, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(12, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(5, v1, 2, 4); - VCMP_I32(6, v2, 6, 8); - - asm volatile("vtdiscard" ::: "memory"); -} - -// ------------------------------------------------------- -// TC4: vtmmu tile isolation — mt0 and mt4 independent -// ------------------------------------------------------- -void TEST_CASE4(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - - VSET(2, e8, m1); - VLOAD_8(v8, 1, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v10, 2, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v24, 5, 7, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v26, 6, 8, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 1, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v18, 0, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtzero mt4" ::: "memory"); - asm volatile("vtmmu.tvv mt0, v8, v16" ::: "memory"); - asm volatile("vtmmu.tvv mt4, v24, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 0); asm volatile("vtmv.v.t v3, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(4, 1); asm volatile("vtmv.v.t v4, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(7, v1, 1, 2); - VCMP_I32(8, v2, 3, 4); - VCMP_I32(9, v3, 5, 6); - VCMP_I32(10, v4, 7, 8); - - asm volatile("vtdiscard" ::: "memory"); -} - -// TC5: vtmmu altfmt=1 (uint8 × int8), A*I=A -void TEST_CASE5(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - +#define LOAD_A(a0, a1, a2, a3) \ + do { \ + VLOAD_8(v8, a0, a2, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + VLOAD_8(v10, a1, a3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + VLOAD_8(v12, 0, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + VLOAD_8(v14, 0, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + } while (0) + +#define LOAD_B(b0, b1, b2, b3) \ + do { \ + VLOAD_8(v16, b0, b1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + VLOAD_8(v18, b2, b3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + VLOAD_8(v20, 0, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + VLOAD_8(v22, 0, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); \ + } while (0) + +#define CONFIG_I8(altfmt) \ + do { \ + asm volatile("msetmtypei 3, 0, " #altfmt ::: "memory"); \ + asm volatile("msettn zero, %0" :: "r"((uintptr_t)2) : "memory"); \ + asm volatile("msettm zero, %0" :: "r"((uintptr_t)2) : "memory"); \ + asm volatile("msettk zero, %0" \ + :: "r"((uintptr_t)KMAX(8, 4)) : "memory"); \ + } while (0) + +#define CONFIG_I32_RESULT() \ + do { \ + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); \ + asm volatile("msettn zero, %0" :: "r"((uintptr_t)2) : "memory"); \ + asm volatile("msettm zero, %0" :: "r"((uintptr_t)2) : "memory"); \ + } while (0) + +// vtmmu, altfmt=0: uint8 * uint8 -> int32. +static void TEST_CASE1(void) { VSET(2, e8, m1); - VLOAD_8(v8, 1, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A - VLOAD_8(v10, 2, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 A - VLOAD_8(v16, 1, 0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B int8 - VLOAD_8(v18, 0, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=1 B int8 - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); + LOAD_A(1, 2, 3, 4); + LOAD_B(5, 7, 6, 8); + CONFIG_I8(0); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); asm volatile("vtmmu.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(11, v1, 1, 2); - VCMP_I32(12, v2, 3, 4); - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("vtdiscard" ::: "memory"); -} - -// TC6: vtmmu altfmt=1 (uint8 × int8), B with negatives -void TEST_CASE6(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; - - VSET(2, e8, m1); - VLOAD_8(v8, 2, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 A (uint8) - VLOAD_8(v16, 1, 0xFE, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // k=0 B (int8: [1, -2]) - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("vtmmu.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(13, v1, 2, -4); - VCMP_I32(14, v2, 3, -6); - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); + CONFIG_I32_RESULT(); + uintptr_t tss = TSS_ROW(0, 0); + asm volatile("vtmv.v.t v1, %0" :: "r"(tss) : "memory"); + tss = TSS_ROW(0, 1); + asm volatile("vtmv.v.t v2, %0" :: "r"(tss) : "memory"); + VCMP_I32(1, v1, 17, 23); + VCMP_I32(2, v2, 39, 53); asm volatile("vtdiscard" ::: "memory"); } -// ------------------------------------------------------- -// TC7: vtmmu altfmt isolation — B=0xFE as uint8 vs int8 -// altfmt=0: C[0][1]=508 altfmt=1: C[0][1]=-4 -// ------------------------------------------------------- -void TEST_CASE7(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; - - // --- altfmt=0: B as uint8 --- +// vtmmu, altfmt=1: uint8 * int8 -> int32. +static void TEST_CASE2(void) { VSET(2, e8, m1); - VLOAD_8(v8, 2, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 1, 0xFE, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // 0xFE=254 as uint8 - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); + LOAD_A(1, 2, 3, 4); + LOAD_B(0xfb, 0xf9, 0xfa, 0xf8); + CONFIG_I8(1); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtmmu.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(15, v1, 2, 508); - VCMP_I32(16, v2, 3, 762); - - // --- altfmt=1: B as int8 --- - VSET(2, e8, m1); - VLOAD_8(v8, 2, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 1, 0xFE, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); // 0xFE=-2 as int8 - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("vtmmu.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(17, v1, 2, -4); - VCMP_I32(18, v2, 3, -6); - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("vtdiscard" ::: "memory"); -} - -// TC8: vtmms int8*uint8 (ALTFMT=0) — C[0]=[-17,-23], C[1]=[39,53] -void TEST_CASE8(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - - VSET(2, e8, m1); - VLOAD_8(v8, 0xFF, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v10, 0xFE, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 5, 7, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v18, 6, 8, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtmms.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(19, v1, -17, -23); - VCMP_I32(20, v2, 39, 53); - - asm volatile("vtdiscard" ::: "memory"); -} - -// TC9: vtmms int8*int8 (ALTFMT=1) — C[0]=[17,23], C[1]=[-39,-53] -void TEST_CASE9(void) { - const uintptr_t TN = 2, TM = 2, TK = 2; - - VSET(2, e8, m1); - VLOAD_8(v8, 0xFF, 3, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v10, 0xFE, 4, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 0xFB, 0xF9, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v18, 0xFA, 0xF8, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("vtmms.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(21, v1, 17, 23); - VCMP_I32(22, v2, -39, -53); - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); + CONFIG_I32_RESULT(); + uintptr_t tss = TSS_ROW(0, 0); + asm volatile("vtmv.v.t v1, %0" :: "r"(tss) : "memory"); + tss = TSS_ROW(0, 1); + asm volatile("vtmv.v.t v2, %0" :: "r"(tss) : "memory"); + VCMP_I32(3, v1, -17, -23); + VCMP_I32(4, v2, -39, -53); asm volatile("vtdiscard" ::: "memory"); } -// TC10: vtmms extreme values — sign extension check, A=[-128,127]*B=[1,1] -void TEST_CASE10(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; - +// vtmms, altfmt=0: int8 * uint8 -> int32. +static void TEST_CASE3(void) { VSET(2, e8, m1); - VLOAD_8(v8, 0x80, 0x7F, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 1, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); + LOAD_A(0xff, 0xfe, 0xfd, 0xfc); + LOAD_B(5, 7, 6, 8); + CONFIG_I8(0); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); asm volatile("vtmms.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(23, v1, -128, -128); - VCMP_I32(24, v2, 127, 127); - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); + CONFIG_I32_RESULT(); + uintptr_t tss = TSS_ROW(0, 0); + asm volatile("vtmv.v.t v1, %0" :: "r"(tss) : "memory"); + tss = TSS_ROW(0, 1); + asm volatile("vtmv.v.t v2, %0" :: "r"(tss) : "memory"); + VCMP_I32(5, v1, -17, -23); + VCMP_I32(6, v2, -39, -53); asm volatile("vtdiscard" ::: "memory"); } -// ------------------------------------------------------- -// TC11: vtmms ALTFMT isolation — B=[0xFF,1]: uint8=255 vs int8=-1 -// ALTFMT=0: C[0]=[255,1] ALTFMT=1: C[0]=[-1,1] -// ------------------------------------------------------- -void TEST_CASE11(void) { - const uintptr_t TN = 2, TM = 2, TK = 1; - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); - - // --- run 1: ALTFMT=0, B is uint8 --- +// vtmms, altfmt=1: int8 * int8 -> int32; read the result by columns. +static void TEST_CASE4(void) { VSET(2, e8, m1); - VLOAD_8(v8, 1, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 0xFF, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); + LOAD_A(0xff, 0xfe, 0xfd, 0xfc); + LOAD_B(0xfb, 0xf9, 0xfa, 0xf8); + CONFIG_I8(1); asm volatile("vtzero mt0" ::: "memory"); asm volatile("vtmms.tvv mt0, v8, v16" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t tss; - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(25, v1, 255, 1); - VCMP_I32(26, v2, 255, 1); - - // --- run 2: ALTFMT=1, B is int8 --- - VSET(2, e8, m1); - VLOAD_8(v8, 1, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - VLOAD_8(v16, 0xFF, 1, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("csrs 0xC21, %0" :: "r"(256) : "memory"); - asm volatile("vtmms.tvv mt0, v8, v16" ::: "memory"); - - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[t]" :: [t]"r"(tss) : "memory"); - tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[t]" :: [t]"r"(tss) : "memory"); - - VCMP_I32(27, v1, -1, 1); - VCMP_I32(28, v2, -1, 1); - - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); + CONFIG_I32_RESULT(); + uintptr_t tss = TSS_COL(0, 0); + asm volatile("vtmv.v.t v1, %0" :: "r"(tss) : "memory"); + tss = TSS_COL(0, 1); + asm volatile("vtmv.v.t v2, %0" :: "r"(tss) : "memory"); + VCMP_I32(7, v1, 17, 39); + VCMP_I32(8, v2, 23, 53); asm volatile("vtdiscard" ::: "memory"); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); - asm volatile("csrc 0xC21, %0" :: "r"(256) : "memory"); - TEST_CASE1(); TEST_CASE2(); TEST_CASE3(); TEST_CASE4(); - TEST_CASE5(); - TEST_CASE6(); - TEST_CASE7(); - TEST_CASE8(); - TEST_CASE9(); - TEST_CASE10(); - TEST_CASE11(); - EXIT_CHECK(); } diff --git a/sw/vme/isaTests/vtmv.c b/sw/vme/isaTests/vtmv.c index 6048e3a31..8f42bd19f 100644 --- a/sw/vme/isaTests/vtmv.c +++ b/sw/vme/isaTests/vtmv.c @@ -2,17 +2,13 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// vtmv.t.v — Move vector registers → tile subset -// vtmv.v.t — Move tile subset → vector registers -// -// ------------------------------------------------------- // Test cases: -// TC1: vtmv.t.v (vector → tile row) then vtmv.v.t read-back → verify round-trip -// TC2: vtmv.t.v into multiple rows of same tile → verify row independence -// TC3: vtmv.t.v into two different tile registers → verify tile independence -// TC4: vtmv.t.v 2x2 tile round-trip with TEW=32 +// TC1: round-trip all rows of a 4x4 tile. +// TC2: preserve zeroed rows while updating non-consecutive rows. +// TC3: keep two tile registers independent. +// TC4: round-trip a 2x2 tile. #include #include "vector_macros.h" @@ -28,7 +24,7 @@ void TEST_CASE1(void) { VLOAD_32(v12, 11, 22, 33, 44, 0,0,0,0, 0,0,0,0, 0,0,0,0); VLOAD_32(v14, 55, 66, 77, 88, 0,0,0,0, 0,0,0,0, 0,0,0,0); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("vtzero mt0" ::: "memory"); @@ -39,7 +35,7 @@ void TEST_CASE1(void) { tss = TSS_ROW(0, 2); asm volatile("vtmv.t.v %[tss], v12" :: [tss]"r"(tss) : "memory"); tss = TSS_ROW(0, 3); asm volatile("vtmv.t.v %[tss], v14" :: [tss]"r"(tss) : "memory"); - VSET(4, e32, m1); + // Preserve matrix configuration while reading the tile back. tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[tss]" :: [tss]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[tss]" :: [tss]"r"(tss) : "memory"); tss = TSS_ROW(0, 2); asm volatile("vtmv.v.t v3, %[tss]" :: [tss]"r"(tss) : "memory"); @@ -61,7 +57,7 @@ void TEST_CASE2(void) { VLOAD_32(v8, 1, 2, 3, 4, 0,0,0,0, 0,0,0,0, 0,0,0,0); VLOAD_32(v10, 9,10,11,12, 0,0,0,0, 0,0,0,0, 0,0,0,0); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("vtzero mt0" ::: "memory"); @@ -70,7 +66,7 @@ void TEST_CASE2(void) { tss = TSS_ROW(0, 0); asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); tss = TSS_ROW(0, 2); asm volatile("vtmv.t.v %[tss], v10" :: [tss]"r"(tss) : "memory"); - VSET(4, e32, m1); + // Preserve matrix configuration while reading the tile back. tss = TSS_ROW(0, 0); asm volatile("vtmv.v.t v1, %[tss]" :: [tss]"r"(tss) : "memory"); tss = TSS_ROW(0, 1); asm volatile("vtmv.v.t v2, %[tss]" :: [tss]"r"(tss) : "memory"); tss = TSS_ROW(0, 2); asm volatile("vtmv.v.t v3, %[tss]" :: [tss]"r"(tss) : "memory"); @@ -92,7 +88,7 @@ void TEST_CASE3(void) { VLOAD_32(v8, 0xA, 0xB, 0xC, 0xD, 0,0,0,0, 0,0,0,0, 0,0,0,0); VLOAD_32(v10, 0x1, 0x2, 0x3, 0x4, 0,0,0,0, 0,0,0,0, 0,0,0,0); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("vtzero mt0" ::: "memory"); @@ -120,7 +116,7 @@ void TEST_CASE4(void) { VLOAD_32(v8, 100, 200, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); VLOAD_32(v10, 300, 400, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("vtzero mt0" ::: "memory"); @@ -141,8 +137,6 @@ void TEST_CASE4(void) { int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); TEST_CASE1(); TEST_CASE2(); diff --git a/sw/vme/isaTests/vtse.c b/sw/vme/isaTests/vtse.c index 441c0ba9d..9676e7252 100644 --- a/sw/vme/isaTests/vtse.c +++ b/sw/vme/isaTests/vtse.c @@ -2,267 +2,156 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// vtse.c — ISA tests for vtse8, vtse16, vtse32, vtse64 -// -// ------------------------------------------------------- // Test cases: -// TC1: vtse32 row store — int32 stored directly -// TC2: vtse8 row store — truncate to uint8 -// TC3: vtse16 row store — truncate to uint16 -// TC4: vtse32 col store — pattern=1 -// TC5: vtse8 all rows of 4×4 tile -// TC6: vtse32 tile isolation (mt0 stored, mt4 untouched) -// TC7: vtmmu then vtse32 — store computed result -// TC8: vtse8 truncation check — values > 255 truncated +// TC1-TC2: 8-bit row and column stores. +// TC3-TC4: 16-bit row and column stores. +// TC5-TC6: 32-bit row and column stores. #include #include "vector_macros.h" #include "vector_matrix_macros.h" -// TC1: vtse32 row store — int32 stored directly -void TEST_CASE1(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - +static void TEST_CASE1(void) { + uintptr_t n = 4, vl; uintptr_t tss = TSS_ROW(0, 0); - + uint8_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint8_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint8_t)(~(uint8_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 0, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(10, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - VLOAD_32(v8, 111, 222, 333, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static int32_t dst[4] = {0, 0, 0, 0}; - uintptr_t base = (uintptr_t)dst; - asm volatile("vtse32 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(4, e32, m1); - asm volatile("vle32.v v1, (%0)" :: "r"(dst) : "memory"); - VCMP_I32(1, v1, 111, 222, 333, 444); - + asm volatile("vle8.v v8, (%0)" :: "r"(src) : "v8", "memory"); + asm volatile("vtmv.t.v %0, v8" :: "r"(tss) : "memory"); + asm volatile("vtse8 %0, (%1)" :: "r"(tss), "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(11, dst[i + 1], src[i]); } + XCMP(12, dst[0], 0x55); + XCMP(13, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC2: vtse8 row store — truncate int32 to low 8 bits -void TEST_CASE2(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - uintptr_t tss = TSS_ROW(0, 0); - +static void TEST_CASE2(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_COL(0, 0); + uint8_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint8_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint8_t)(~(uint8_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 0, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(20, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - VLOAD_32(v8, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static uint8_t dst[4] = {0xFF, 0xFF, 0xFF, 0xFF}; - uintptr_t base = (uintptr_t)dst; - asm volatile("vtse8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(4, e8, m1); - asm volatile("vle8.v v1, (%0)" :: "r"(dst) : "memory"); - VCMP_U8(2, v1, 1, 2, 3, 4); - + asm volatile("vle8.v v8, (%0)" :: "r"(src) : "v8", "memory"); + asm volatile("vtmv.t.v %0, v8" :: "r"(tss) : "memory"); + asm volatile("vtse8 %0, (%1)" :: "r"(tss), "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(21, dst[i + 1], src[i]); } + XCMP(22, dst[0], 0x55); + XCMP(23, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC3: vtse16 row store — truncate int32 to low 16 bits -void TEST_CASE3(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - +static void TEST_CASE3(void) { + uintptr_t n = 4, vl; uintptr_t tss = TSS_ROW(0, 0); - + uint16_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint16_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint16_t)(~(uint16_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 1, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(30, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - VLOAD_32(v8, 100, 200, 300, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static uint16_t dst[4] = {0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF}; - uintptr_t base = (uintptr_t)dst; - asm volatile("vtse16 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(4, e16, m1); - asm volatile("vle16.v v1, (%0)" :: "r"(dst) : "memory"); - VCMP_U16(3, v1, 100, 200, 300, 400); - + asm volatile("vle16.v v8, (%0)" :: "r"(src) : "v8", "memory"); + asm volatile("vtmv.t.v %0, v8" :: "r"(tss) : "memory"); + asm volatile("vtse16 %0, (%1)" :: "r"(tss), "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(31, dst[i + 1], src[i]); } + XCMP(32, dst[0], 0x55); + XCMP(33, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC4: vtse32 col store — pattern=1 -void TEST_CASE4(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - +static void TEST_CASE4(void) { + uintptr_t n = 4, vl; uintptr_t tss = TSS_COL(0, 0); - + uint16_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint16_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint16_t)(~(uint16_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 1, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(40, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - VLOAD_32(v8, 10, 20, 30, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static int32_t dst[4] = {0, 0, 0, 0}; - uintptr_t base = (uintptr_t)dst; - asm volatile("vtse32 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(4, e32, m1); - asm volatile("vle32.v v1, (%0)" :: "r"(dst) : "memory"); - VCMP_I32(4, v1, 10, 20, 30, 40); - + asm volatile("vle16.v v8, (%0)" :: "r"(src) : "v8", "memory"); + asm volatile("vtmv.t.v %0, v8" :: "r"(tss) : "memory"); + asm volatile("vtse16 %0, (%1)" :: "r"(tss), "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(41, dst[i + 1], src[i]); } + XCMP(42, dst[0], 0x55); + XCMP(43, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC5: vtse8 all rows — store all 4 rows to separate buffers -void TEST_CASE5(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - - uintptr_t t; - VLOAD_32(v8, 1, 2, 3, 4, 0,0,0,0,0,0,0,0,0,0,0,0); - t = TSS_ROW(0, 0); asm volatile("vtmv.t.v %[t], v8" :: [t]"r"(t) : "memory"); - VLOAD_32(v8, 5, 6, 7, 8, 0,0,0,0,0,0,0,0,0,0,0,0); - t = TSS_ROW(0, 1); asm volatile("vtmv.t.v %[t], v8" :: [t]"r"(t) : "memory"); - VLOAD_32(v8, 9,10,11,12, 0,0,0,0,0,0,0,0,0,0,0,0); - t = TSS_ROW(0, 2); asm volatile("vtmv.t.v %[t], v8" :: [t]"r"(t) : "memory"); - VLOAD_32(v8,13,14,15,16, 0,0,0,0,0,0,0,0,0,0,0,0); - t = TSS_ROW(0, 3); asm volatile("vtmv.t.v %[t], v8" :: [t]"r"(t) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static uint8_t r0[4], r1[4], r2[4], r3[4]; - uintptr_t tss, base; - - tss = TSS_ROW(0, 0); base = (uintptr_t)r0; - asm volatile("vtse8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - tss = TSS_ROW(0, 1); base = (uintptr_t)r1; - asm volatile("vtse8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - tss = TSS_ROW(0, 2); base = (uintptr_t)r2; - asm volatile("vtse8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - tss = TSS_ROW(0, 3); base = (uintptr_t)r3; - asm volatile("vtse8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(4, e8, m1); - asm volatile("vle8.v v1, (%0)" :: "r"(r0) : "memory"); - VCMP_U8(5, v1, 1, 2, 3, 4); - asm volatile("vle8.v v1, (%0)" :: "r"(r1) : "memory"); - VCMP_U8(6, v1, 5, 6, 7, 8); - asm volatile("vle8.v v1, (%0)" :: "r"(r2) : "memory"); - VCMP_U8(7, v1, 9, 10, 11, 12); - asm volatile("vle8.v v1, (%0)" :: "r"(r3) : "memory"); - VCMP_U8(8, v1, 13, 14, 15, 16); - - asm volatile("vtdiscard" ::: "memory"); -} - -// TC6: vtse32 tile isolation — mt0 stored, mt4 untouched -void TEST_CASE6(void) { - const uintptr_t TN = 6, TM = 6, TK = 2; - +static void TEST_CASE5(void) { + uintptr_t n = 4, vl; uintptr_t tss = TSS_ROW(0, 0); - + uint32_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint32_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint32_t)(~(uint32_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(50, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("vtzero mt4" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - VLOAD_32(v8, 77, 88, 0,0,0,0,0,0,0,0,0,0,0,0,0,0); - asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static int32_t dst0[2] = {-1, -1}; - int32_t *dst4 = (int32_t *)snrt_l1alloc(2 * sizeof(int32_t)); - dst4[0] = -1; dst4[1] = -1; - - uintptr_t base = (uintptr_t)dst0; - asm volatile("vtse32 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(2, e32, m1); - asm volatile("vle32.v v1, (%0)" :: "r"(dst0) : "memory"); - VCMP_I32(9, v1, 77, 88); - asm volatile("vle32.v v1, (%0)" :: "r"(dst4) : "memory"); - VCMP_I32(10, v1, -1, -1); // unchanged - + asm volatile("vle32.v v8, (%0)" :: "r"(src) : "v8", "memory"); + asm volatile("vtmv.t.v %0, v8" :: "r"(tss) : "memory"); + asm volatile("vtse32 %0, (%1)" :: "r"(tss), "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(51, dst[i + 1], src[i]); } + XCMP(52, dst[0], 0x55); + XCMP(53, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } -// TC8: vtse8 truncation — values > 255 lose high bits -// Load [256, 257, 0x1FF, -1], vtse8 → [0, 1, 0xFF, 0xFF] -void TEST_CASE7(void) { - const uintptr_t TN = 4, TM = 4, TK = 4; - - uintptr_t tss = TSS_ROW(0, 0); - +static void TEST_CASE6(void) { + uintptr_t n = 4, vl; + uintptr_t tss = TSS_COL(0, 0); + uint32_t *src = snrt_l1alloc(4 * sizeof(*src)); + uint32_t *dst = snrt_l1alloc(6 * sizeof(*dst)); + for (unsigned i = 0; i < 4; ++i) src[i] = (uint32_t)(~(uint32_t)0 - i); + for (unsigned i = 0; i < 6; ++i) dst[i] = 0x55; + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); + asm volatile("msettn %0, %1" : "=r"(vl) : "r"(n) : "memory"); + XCMP(60, vl, n); + asm volatile("msettm zero, %0" :: "r"(n) : "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - VLOAD_32(v8, 256, 257, 0x1FF, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - asm volatile("vtmv.t.v %[tss], v8" :: [tss]"r"(tss) : "memory"); - - asm volatile("msetmtypei 3, 0" ::: "memory"); - asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); - asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); - asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); - - static uint8_t dst[4] = {0xAA, 0xAA, 0xAA, 0xAA}; - uintptr_t base = (uintptr_t)dst; - asm volatile("vtse8 %[tss], (%[base])" :: [tss]"r"(tss), [base]"r"(base) : "memory"); - - VSET(4, e8, m1); - asm volatile("vle8.v v1, (%0)" :: "r"(dst) : "memory"); - VCMP_U8(12, v1, 0x00, 0x01, 0xFF, 0xFF); - + asm volatile("vle32.v v8, (%0)" :: "r"(src) : "v8", "memory"); + asm volatile("vtmv.t.v %0, v8" :: "r"(tss) : "memory"); + asm volatile("vtse32 %0, (%1)" :: "r"(tss), "r"(dst + 1) : "memory"); + asm volatile("fence rw, rw" ::: "memory"); + for (unsigned i = 0; i < 4; ++i) { XCMP(61, dst[i + 1], src[i]); } + XCMP(62, dst[0], 0x55); + XCMP(63, dst[5], 0x55); asm volatile("vtdiscard" ::: "memory"); } int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); - TEST_CASE1(); TEST_CASE2(); TEST_CASE3(); - // TEST_CASE4(); + TEST_CASE4(); TEST_CASE5(); TEST_CASE6(); - TEST_CASE7(); - EXIT_CHECK(); } diff --git a/sw/vme/isaTests/vtzero.c b/sw/vme/isaTests/vtzero.c index b2010c699..c987dd4a6 100644 --- a/sw/vme/isaTests/vtzero.c +++ b/sw/vme/isaTests/vtzero.c @@ -2,16 +2,13 @@ // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51 // -// Authors: Pei-Yu Lin (peilin@ethz.ch) +// Author: Pei-Yu Lin, ETH Zürich // -// vtzero.c — ISA tests for vtzero -// -// ------------------------------------------------------- // Test cases: -// TC1: FP32, TEW=32, 4×4 — vtfmm then vtzero → zeros -// TC2: FP32, TEW=32, two tiles — vtzero mt0 only, mt4 stays intact -// TC3: FP32, TEW=32, 4×4 — pre-fill with vtfmm, vtzero → zeros -// TC4: FP32, TEW=32, 4×4 — I*I then vtzero → zeros +// TC1: clear a populated tile. +// TC2: clear one tile while preserving another tile. +// TC3: clear a tile populated with repeated rows. +// TC4: clear a tile populated by an identity product. #include #include "vector_macros.h" @@ -37,7 +34,7 @@ // TC1: FP32/TEW=32, 4×4 — vtfmm.tvv then vtzero → all zeros void TEST_CASE1(void) { - const uintptr_t TN = 4, TM = 4, TK = 1; + const uintptr_t TN = 4, TM = 4, TK = KMAX(32, 1); VSET(4, e32, m1); VLOAD_32(v8, F32_1, F32_5, F32_9, F32_13); @@ -50,7 +47,7 @@ void TEST_CASE1(void) { VLOAD_32(v20, F32_0, F32_0, F32_1, F32_0); VLOAD_32(v22, F32_0, F32_0, F32_0, F32_1); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); @@ -58,7 +55,7 @@ void TEST_CASE1(void) { asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -73,7 +70,7 @@ void TEST_CASE1(void) { // TC2: FP32/TEW=32, two tiles — vtzero mt0 only, mt4 stays intact void TEST_CASE2(void) { - const uintptr_t TN = 4, TM = 4, TK = 1; + const uintptr_t TN = 4, TM = 4, TK = KMAX(32, 1); VSET(4, e32, m1); VLOAD_32(v8, F32_1, F32_5, F32_9, F32_13); @@ -86,7 +83,7 @@ void TEST_CASE2(void) { VLOAD_32(v20, F32_0, F32_0, F32_1, F32_0); VLOAD_32(v22, F32_0, F32_0, F32_0, F32_1); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); @@ -95,7 +92,7 @@ void TEST_CASE2(void) { asm volatile("vtfmm.tvv mt4, v8, v16" ::: "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -115,7 +112,7 @@ void TEST_CASE2(void) { // TC3: FP32/TEW=32, 4×4 — pre-fill with vtfmm, vtzero → zeros void TEST_CASE3(void) { - const uintptr_t TN = 4, TM = 4, TK = 1; + const uintptr_t TN = 4, TM = 4, TK = KMAX(32, 1); VSET(4, e32, m1); VLOAD_32(v8, F32_1, F32_2, F32_3, F32_4); @@ -128,7 +125,7 @@ void TEST_CASE3(void) { VLOAD_32(v20, F32_0, F32_0, F32_1, F32_0); VLOAD_32(v22, F32_0, F32_0, F32_0, F32_1); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); @@ -136,7 +133,7 @@ void TEST_CASE3(void) { asm volatile("vtfmm.tvv mt0, v8, v16" ::: "memory"); asm volatile("vtzero mt0" ::: "memory"); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); @@ -151,7 +148,7 @@ void TEST_CASE3(void) { // TC4: FP32/TEW=32, 4×4 — vtfmm I*I then vtzero → all zeros void TEST_CASE4(void) { - const uintptr_t TN = 4, TM = 4, TK = 1; + const uintptr_t TN = 4, TM = 4, TK = KMAX(32, 1); VSET(4, e32, m1); VLOAD_32(v8, F32_1, F32_0, F32_0, F32_0); @@ -164,7 +161,7 @@ void TEST_CASE4(void) { VLOAD_32(v20, F32_0, F32_0, F32_1, F32_0); VLOAD_32(v22, F32_0, F32_0, F32_0, F32_1); - asm volatile("msetmtypei 1, 2" ::: "memory"); + asm volatile("msetmtypei 1, 2, 0" ::: "memory"); asm volatile("msettn x0, %0" :: "r"(TN) : "memory"); asm volatile("msettm x0, %0" :: "r"(TM) : "memory"); asm volatile("msettk x0, %0" :: "r"(TK) : "memory"); @@ -183,8 +180,6 @@ void TEST_CASE4(void) { int main(void) { INIT_CHECK(); - enable_vec(); - enable_vme(); TEST_CASE1(); TEST_CASE2(); @@ -192,4 +187,4 @@ int main(void) { TEST_CASE4(); EXIT_CHECK(); -} \ No newline at end of file +}